framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1,0.00973866693675518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,3,0.009829333052039146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,31,0.009904000287254652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,63,0.009946666657924652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,1,0.01246400053302447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,127,0.04068800061941147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,127,0.012645332763592402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,255,0.014896000425020853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,255,0.014175999909639359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,511,0.016143999993801117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,511,0.01714133347074191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,7,0.009925333162148794
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1023,0.023168000082174938
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,15,0.009839999799927076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,2047,0.043568000197410583
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,4095,0.0415040006240209
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,4095,0.0589279979467392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1,0.013530666629473368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,1,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,3,0.01360000049074491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,3,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,7,0.013679999858140945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,7,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,15,0.013637332866589228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,15,0.012618667135636011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,3,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,31,0.013744000345468521
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,63,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,31,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,63,0.01393066719174385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,127,0.01293333371480306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,63,0.012629333883523941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,127,0.014303999642531076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,255,0.01451733335852623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,255,0.017877332866191864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,511,0.023215999205907185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,511,0.0176959993938605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1023,0.028463999430338543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,7,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,1023,0.023402666052182514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,2047,0.035616000493367515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,2047,0.03846399982770284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,4095,0.059301331639289856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,4095,0.22129599253336588
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1,0.012730666746695837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,3,0.013562666873137156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1,0.013642666240533194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,3,0.012565333396196365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,7,0.013663999736309052
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,1023,0.023200000325838726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,7,0.012629333883523941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,15,0.013765333841244379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,15,0.012784000486135483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,31,0.013765333841244379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,63,0.013936000565687815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,63,0.012842666357755661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,2047,0.03527999917666117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,31,0.012693333129088083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,127,0.013290667285521826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,127,0.014607999473810196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,255,0.022458667556444805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,255,0.014970666418472925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,511,0.027514666318893433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,511,0.01811733345190684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,2047,0.20651199420293173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,2047,0.03676266719897588
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1023,0.03707200040419897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1023,0.024271999796231587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,4095,0.3951573371887207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1,0.014848000059525171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,4095,0.19510400295257568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1,0.017024000485738117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,15,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,3,0.014639999717473984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,7,0.017360000560681026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,3,0.017386666188637417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,7,0.014767999450365702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,15,0.014720000326633453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,15,0.01736533393462499
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,31,0.012367999802033106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,31,0.014746667196353277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,127,0.019066666563351948
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,127,0.016095999628305435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,63,0.01782400036851565
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,63,0.015178666760524115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,31,0.017605333278576534
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,255,0.023951999843120575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1023,0.2015733321507772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1023,0.03448000053564707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,255,0.01894933357834816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,511,0.02401600033044815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,511,0.03377600014209747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,2047,0.42846934000651044
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,2047,0.1976426641146342
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1,0.008746666833758354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,4095,0.37539199988047284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,7,0.0107893335322539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,7,0.009093333035707474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,15,0.00897066667675972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,3,0.008799999952316284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,15,0.010666667173306147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,4095,0.7578293482462565
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,63,0.009088000282645226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,31,0.010832000523805618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,31,0.00902399979531765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,63,0.010709332923094431
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,3,0.010735999792814255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,127,0.009232000137368837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,511,0.011770666887362799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,127,0.01089599976936976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,255,0.011695999652147293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,255,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,511,0.01293333371480306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1023,0.013728000223636627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,2047,0.014789332946141561
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,1023,0.011594666788975397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,2047,0.011994666109482447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,4095,0.012213333199421564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,8191,0.013701333353916803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,8191,0.02179733415444692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,4095,0.01717866708834966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,16383,0.0316746657093366
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,16383,0.016735999534527462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1,0.008799999952316284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,1,0.010805333654085795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,3,0.00877333308259646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,3,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,7,0.008885333314538002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,31,0.008890666688481966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,15,0.010725333044926325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,63,0.008842666943868002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,15,0.008837333569924036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,127,0.009237333511312803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,255,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,511,0.012757333616415659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,255,0.0116799995303154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,511,0.011909333368142446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1023,0.01332266628742218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,1023,0.011781333635250727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,2047,0.014533333480358124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,4095,0.017018667111794155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,2047,0.011957333733638128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,4095,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,8191,0.02186133215824763
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,16383,0.03566933423280716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,8191,0.014053333550691605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1,0.008672000219424566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,16383,0.01695999999841054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,3,0.008709333216150602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1,0.010901333143313726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,15,0.008810666700204214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,31,0.008832000195980072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,15,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,7,0.009082666908701261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,63,0.00895999992887179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,31,0.010847999403874079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,127,0.009119999905427298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,63,0.010832000523805618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,255,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,511,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,511,0.011941333611806234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,255,0.011941333611806234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1023,0.013258667041858038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,2047,0.014618666221698126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1023,0.01211200033624967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,2047,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,4095,0.01747200017174085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,4095,0.012949333836634954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,8191,0.026485333840052288
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,16383,0.03164266546567281
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,8191,0.01462399959564209
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1,0.0086666668454806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,16383,0.017680000513792038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1,0.011354666203260422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,3,0.01108266661564509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,7,0.011584000041087469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,3,0.008762666955590248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,15,0.01118933285276095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,31,0.008901333436369896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,7,0.008826666822036108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,15,0.008789333204428354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,63,0.008954666554927826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,31,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,127,0.00927466650803884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,127,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,255,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,63,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,511,0.012624000509579977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1023,0.01349866638580958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,511,0.014074667046467463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,255,0.012954667210578918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,4095,0.021898667017618816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1023,0.01421333352724711
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,8191,0.02699200063943863
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,2047,0.015279999623696009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,4095,0.015541333705186844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,2047,0.01481066644191742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,16383,0.036506667733192444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,8191,0.01841066653529803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,16383,0.02351466566324234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1,0.009072000160813332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,3,0.009114666531483332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,7,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,3,0.010794666906197866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,15,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,7,0.010879999647537867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,15,0.01090666651725769
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,31,0.009114666531483332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,63,0.009082666908701261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,31,0.010890666395425797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,127,0.009248000259200731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,63,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,511,0.013114667187134424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,127,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,255,0.011887999872366587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,255,0.012757333616415659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,511,0.011861333002646765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1023,0.013701333353916803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,2047,0.014794666320085526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,1023,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,4095,0.01748266691962878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,2047,0.012229333321253458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,8191,0.022848000129063923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,4095,0.013631999492645264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,8191,0.016832000265518825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,16383,0.03760000069936117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1,0.012128000458081564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,16383,0.022624000906944275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,1,0.011007999380429586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,3,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,7,0.012181332955757776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,3,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,15,0.012128000458081564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,15,0.01099733387430509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,31,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,63,0.01246400053302447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,127,0.012527999778588613
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,127,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,255,0.014858666807413101
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,255,0.012042666474978128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,511,0.015301333119471868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,511,0.011882666498422623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1023,0.015930666277805965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,1023,0.012047999848922094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,2047,0.01716800034046173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,4095,0.020175999651352566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,2047,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,4095,0.013877333452304205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,8191,0.0295413335164388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,8191,0.01700266698996226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,16383,0.034688000877698265
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,16383,0.023029332359631855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1,0.012223999947309494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,3,0.012170666207869848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,7,0.0122079998254776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,3,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,15,0.012357333054145178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,31,0.012256000190973282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,7,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,15,0.011045332998037338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,63,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,31,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,127,0.012527999778588613
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,63,0.011125333607196808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,255,0.014949332922697067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,127,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,511,0.015125333021084467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,255,0.012005332857370377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,511,0.01211200033624967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1023,0.016143999993801117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,2047,0.017717332889636356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1023,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,4095,0.024645333488782246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,2047,0.012693333129088083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,8191,0.029877332349618275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,4095,0.014357333381970724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,8191,0.01775466650724411
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,16383,0.03948266555865606
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,16383,0.02385066697994868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1,0.012106666962305704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,7,0.012229333321253458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,15,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,3,0.011354666203260422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,7,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,3,0.012165332833925882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,31,0.012240000069141388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,15,0.011381333072980246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,31,0.011391999820868174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,63,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,63,0.01166933278242747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,127,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,127,0.01157333329319954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,511,0.01543466622630755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,255,0.015200000256299973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1023,0.016496000190575916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,255,0.014165333161751429
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,511,0.0143306665122509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1023,0.014576000471909841
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,2047,0.022319999833901722
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,2047,0.015466666469971338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,4095,0.027322667340437572
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,4095,0.01826133330663045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,8191,0.023584000766277313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,16383,0.03342933456103007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,16383,0.22340800364812216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,8191,0.03738133360942205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,1,0.013248000293970108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1,0.009824000298976898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,3,0.009749333063761393
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,3,0.013264000415802002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,7,0.009962666779756546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,7,0.013397333522637686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,15,0.00996800015370051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,15,0.013370666652917862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,31,0.009930666536092758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,31,0.013338666409254074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,63,0.010202666744589806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,63,0.013728000223636627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,127,0.010559999694426855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,127,0.013722666849692663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,255,0.016864000509182613
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,255,0.015141333142916361
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,511,0.018394666413466137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,511,0.023045333723227184
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1023,0.028522667785485584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,1023,0.024405332903067272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,2047,0.03808533400297165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,2047,0.03616533428430557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1,0.014549333602190018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,1,0.01351999988158544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,3,0.014607999473810196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,3,0.013493333011865616
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,31,0.014831999937693277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,15,0.014682666709025701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,7,0.014565333724021912
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,7,0.013280000537633896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,15,0.013461332768201828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,31,0.01357866699496905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,63,0.015087999403476715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,63,0.013712000101804733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,127,0.015781333049138386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,127,0.014192000031471252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,255,0.01704000060757001
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,255,0.02499199906984965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,511,0.018874666343132656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,511,0.030031998952229817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1,0.015402667224407196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,1023,0.025146665672461193
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,3,0.01815466706951459
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1023,0.03996799886226654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,3,0.015509333461523056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1,0.017968000223239262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,7,0.01788266624013583
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,2047,0.0374293327331543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,2047,0.20759467283884683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,15,0.018346666047970455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,7,0.015696000307798386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,31,0.01850133389234543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,15,0.015439999600251516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,31,0.01568000018596649
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,63,0.018751999984184902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,127,0.017093333105246227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,255,0.025029333929220837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,255,0.019530666371186573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,511,0.02478933334350586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,63,0.01624533285697301
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,127,0.019978666057189304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,2047,0.2009013295173645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,511,0.0347626656293869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,2047,0.3951946496963501
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1023,0.20441599686940512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1023,0.03517866631348928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1,0.021514666577180225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,3,0.02536533276240031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,7,0.021568000316619873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,3,0.021397332350413006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1,0.025098666548728943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,31,0.021690666675567627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,31,0.025818665822347004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,7,0.025477332373460133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,63,0.026352000733216602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,63,0.0220266655087471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,127,0.027834666272004444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,15,0.025461333493391674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,127,0.023455999791622162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,15,0.02162133405605952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,255,0.02845333268245061
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1023,0.19923200209935507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,255,0.037903999288876854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1023,0.39574400583902997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,2047,0.764677365620931
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,511,0.20644799868265787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1,0.008965333302815756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,511,0.038191998998324074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,1,0.011168000598748526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,3,0.009039999917149544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,7,0.011071999867757162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,3,0.011194666226704916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,15,0.011039999624093374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,31,0.00922133338948091
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,31,0.01101333275437355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,63,0.011194666226704916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,63,0.00926399976015091
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,7,0.009130666653315226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,2047,0.3804159959157308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,255,0.012960000584522883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,127,0.00955200009047985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,127,0.011168000598748526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,255,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,511,0.013301332791646322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1023,0.013978666315476099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,511,0.012170666207869848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,2047,0.015301333119471868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,1023,0.012245333443085352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,2047,0.013829333086808523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,4095,0.018613333503405254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,15,0.009029333169261614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,4095,0.016895999511082966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,8191,0.02861333390076955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,8191,0.022709332406520844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,16383,0.03363200028737386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,16383,0.03484266748030981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1,0.012495999534924826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,1,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,3,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,7,0.012538666526476542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,15,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,7,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,3,0.011141333729028702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,15,0.011317333827416102
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,31,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,63,0.01268799975514412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,31,0.012485332787036896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,127,0.012938667088747025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,63,0.011285333583752314
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,255,0.017130666722853977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,127,0.011285333583752314
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,255,0.012138667205969492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,511,0.015466666469971338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,511,0.012138667205969492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1023,0.016506666938463848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,1023,0.012432000289360682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,2047,0.0182239996890227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,4095,0.02553066611289978
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,2047,0.013845333208640417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,4095,0.016997333616018295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,8191,0.030378667016824085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,8191,0.023013333479563396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,16383,0.04040000090996424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,3,0.01249066616098086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,16383,0.03522133330504099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,3,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,7,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,15,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,31,0.01267733300725619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,15,0.01121066634853681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,7,0.012469333906968435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,63,0.012666666259368261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,31,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,127,0.012890666723251343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,63,0.011402666568756104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,127,0.0116799995303154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,255,0.015066667149464289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,511,0.015583999454975128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,511,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,255,0.012469333906968435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1023,0.01674666628241539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,2047,0.022624000906944275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1023,0.013061333447694778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,4095,0.02996266633272171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,2047,0.014485333114862442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,8191,0.03739733248949051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,4095,0.01758933315674464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,8191,0.023872000475724537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,16383,0.20532800753911337
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,16383,0.03625066578388214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,7,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1,0.013072000195582708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,3,0.013173333058754602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,15,0.012655999511480331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,3,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,7,0.013088000317414602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,15,0.01322666679819425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,31,0.012576000144084295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,63,0.013232000172138214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,255,0.01545599972208341
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,127,0.013631999492645264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,127,0.012885333349307379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,31,0.013082666943470636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,63,0.012794667234023413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,511,0.01621866722901662
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1023,0.021066665649414062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,511,0.014922666052977243
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,255,0.014831999937693277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,4095,0.04060266663630804
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1023,0.015722667177518208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,2047,0.026357332865397137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,2047,0.01838933303952217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,4095,0.02367999901374181
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,8191,0.20227199792861938
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,8191,0.03398400048414866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,16383,0.19475199778874716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,16383,0.39076801141103107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,1,0.015013333410024643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1,0.010559999694426855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,3,0.010656000425418219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,3,0.015077333897352219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,7,0.014965333044528961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,7,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,15,0.010922666639089584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,31,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,15,0.015066667149464289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,31,0.015024000157912573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,63,0.01108266661564509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,63,0.015253332753976187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,127,0.011711999773979187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,127,0.01587733378012975
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,255,0.017221332838137943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,255,0.026378666361172993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,511,0.03126399964094162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,511,0.020442667106787365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1,0.020074666788180668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1023,0.041152000427246094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,1023,0.026736001173655193
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,1,0.017082666357358296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,3,0.019989332805077236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,3,0.017055999487638474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,7,0.020266667008399963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,7,0.017184000462293625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,15,0.02029866725206375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,15,0.0170666662355264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,31,0.020474666108687718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,31,0.01721599946419398
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,63,0.020687999824682873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,63,0.017674667139848072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,127,0.018570666511853535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,127,0.022282667458057404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,255,0.026943999032179516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,255,0.021162666380405426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,511,0.03690666705369949
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,511,0.026170666019121807
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1,0.02696000039577484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1,0.023354666928450268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,1023,0.036533333361148834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1023,0.20348266760508218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,3,0.02717866748571396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,3,0.023290666441122692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,7,0.027424000203609467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,15,0.02756800005833308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,15,0.023402666052182514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,7,0.023152001202106476
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,63,0.02809600035349528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,63,0.02383466561635335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,127,0.029861333469549816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,127,0.025285333395004272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,31,0.027674667537212372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,31,0.023247999449570973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,255,0.04004266609748205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,255,0.03050133337577184
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1,0.04051200052102407
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1,0.03455466777086258
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,511,0.040037333965301514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,511,0.2116373380025228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,7,0.04171733558177948
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,7,0.03443199892838796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1023,0.20601600408554077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,3,0.04124800115823746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1023,0.4050346612930298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,3,0.034490667283535004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,31,0.041936000188191734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,31,0.03458133339881897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,63,0.035002666215101876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,15,0.04152533411979675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,15,0.03473600000143051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,127,0.035760000348091125
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1023,0.8537546793619791
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,127,0.04526400069395701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,63,0.0429013321797053
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,255,0.21245867013931274
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,255,0.046015997727712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1023,0.38838934898376465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,511,0.4066186745961507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,511,0.2061013380686442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,1,0.015573333948850632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,3,0.015802666544914246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,7,0.020597333709398907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,1,0.02027733375628789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,3,0.020373333245515823
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,15,0.02049066623051961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,7,0.015861333658297855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,15,0.016048000504573185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,31,0.016373333831628162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,31,0.020479999482631683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,63,0.016544000556071598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,127,0.01800000046690305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,63,0.02082666630546252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,127,0.021781332790851593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,255,0.022805333137512207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,255,0.024559999505678814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,1,0.030997333427270252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,1,0.02645866572856903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,3,0.03146133323510488
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,7,0.031632001201311745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,3,0.0266239990790685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,7,0.026629333694775898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,15,0.03188266605138779
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,15,0.026474667092164356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,31,0.03203733265399933
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,31,0.026560001075267792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,63,0.0323786661028862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,63,0.027306665976842243
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,127,0.034490667283535004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,127,0.028853334486484528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,255,0.03350933392842611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,255,0.04452799757321676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,1,0.038373333712418876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,1,0.04470933477083842
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,3,0.045461331804593406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,3,0.03834133346875509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,7,0.045935998360315956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,7,0.03852266569932302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,15,0.045653333266576133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,15,0.03839999934037527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,31,0.04643733302752177
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,31,0.03862933317820231
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,63,0.046869332591692604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,63,0.03896533449490865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,127,0.049413333336512245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,127,0.04061333338419596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,1,0.06927466889222463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,1,0.057717333237330117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,255,0.22432533899943033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,255,0.050704002380371094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,3,0.07022400200366974
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,3,0.057850668827692665
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,15,0.07117866476376851
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,7,0.07156800230344136
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,7,0.05794133245944977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,15,0.05782933533191681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,63,0.07354666789372762
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,63,0.058549334605534874
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,31,0.07189333438873291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,31,0.058133333921432495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,127,0.2257173260052999
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,127,0.06097066899140676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,255,0.4236746629079183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,255,0.2185866634051005
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,1,0.03349866718053818
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,1,0.025706666211287182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,3,0.03333866596221924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,3,0.025861332813898723
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,7,0.03365333378314972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,7,0.02603200078010559
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,31,0.026352000733216602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,31,0.03385066737731298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,63,0.026965332527955372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,63,0.0340693344672521
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,15,0.033546666304270424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,15,0.02601066728432973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,127,0.029157333076000214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,127,0.03544000039498011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,1,0.0524533341328303
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,1,0.045221333702405296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,3,0.05277866621812185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,3,0.04530666768550873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,7,0.053247998158137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,7,0.045509333411852516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,15,0.05365333457787832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,15,0.04535466432571411
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,31,0.054272000988324486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,31,0.045226668318112694
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,63,0.05450133482615153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,63,0.045850664377212524
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,127,0.0684799998998642
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,127,0.04698666433493296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,1,0.07778666913509369
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,1,0.06675200164318085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,3,0.06685866912206014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,3,0.07875733574231465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,7,0.07956266899903615
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,7,0.0668693333864212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,15,0.0796800007422765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,15,0.06695466736952464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,31,0.08073066671689351
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,31,0.06723199784755707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,63,0.09194667140642802
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,63,0.06791466474533081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,127,0.2526826659838359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,127,0.07434666653474171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,1,0.1284213364124298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,1,0.10610666871070862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,3,0.12981333335240683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,3,0.1060746709505717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,7,0.13205333550771078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,15,0.10636799534161885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,15,0.13173866271972656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,31,0.1397546629110972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,7,0.1058026651541392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,31,0.10657599568367004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,3,0.009258666386206945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,1,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1,0.009173333023985228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,7,0.00927466650803884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,7,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,15,0.009279999881982803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,31,0.009408000235756239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,15,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,31,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,3,0.011338666081428528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,63,0.009535999968647957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,63,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,127,0.009610666582981745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,127,0.01146666705608368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,255,0.013248000293970108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,255,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,511,0.013503999759753546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,511,0.012613333761692047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1023,0.014266667266686758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,1023,0.014111999422311783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,63,0.2545493245124817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,2047,0.016538667182127636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,2047,0.017194667210181553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,63,0.11102933684984843
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,4095,0.02454400062561035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,4095,0.022997332115968067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,8191,0.035029334326585136
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,16383,0.03947199881076813
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,16383,0.05902933577696482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1,0.012719999998807907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,3,0.011338666081428528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,3,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,8191,0.029813334345817566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,7,0.01126933346192042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,15,0.012863999853531519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,1,0.011381333072980246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,15,0.011429333438475927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,31,0.013077333569526672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,31,0.011509332805871964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,63,0.011594666788975397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,63,0.013050666699806849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,255,0.015722667177518208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,127,0.01157333329319954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,255,0.01267733300725619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,127,0.013178666432698568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,511,0.01609066625436147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,511,0.012762666990359625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1023,0.017477333545684814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,127,0.4584266742070516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,1023,0.014384000251690546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,2047,0.023520000278949738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,127,0.24897066752115884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,4095,0.02865600089232127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,2047,0.017263999829689663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,4095,0.023408000667889912
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,8191,0.038592000802357994
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1,0.012863999853531519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,16383,0.20150933663050333
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,16383,0.05931200087070465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,7,0.013023999830087027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,3,0.011482667177915573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,7,0.011578666667143503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,15,0.011653333902359009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,3,0.012879999975363413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,31,0.013253333667914072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,15,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,63,0.013013333082199097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,31,0.011509332805871964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,63,0.011717333147923151
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,127,0.013178666432698568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,127,0.011829332758982977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,255,0.015626666446526844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,255,0.01301866645614306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,511,0.01350933313369751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,511,0.01653333380818367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,2047,0.026778665681680042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1023,0.015034666905800501
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,2047,0.017968000223239262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1023,0.02165333429972331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,4095,0.040949332217375435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,8191,0.20383999745051065
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,4095,0.024186665813128155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1,0.012944000462690989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,8191,0.036874666810035706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,16383,0.39498666922251385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,16383,0.19338132937749228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,8191,0.035616000493367515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1,0.0116799995303154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1,0.013424000392357508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,7,0.012917333592971167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,3,0.01351999988158544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,3,0.012847999731699625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,15,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,7,0.013370666652917862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,15,0.013440000514189402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,31,0.012938667088747025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,63,0.013221333424250284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,31,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,127,0.013733333597580591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,63,0.013541333377361298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,255,0.016314666718244553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1023,0.02611200014750163
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,511,0.02124800036350886
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,127,0.01403733342885971
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,511,0.016127999871969223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,255,0.01524266724785169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,2047,0.035887998839219414
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1023,0.01855466639002164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,2047,0.024101334313551586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,4095,0.20225600401560465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,8191,0.39052267869313556
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,4095,0.034160000582536064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,8191,0.19507733980814615
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,16383,0.37281068166097003
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,16383,0.7582240104675293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,1,0.04473066826661428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,1,0.036389333506425224
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,3,0.045312002301216125
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,3,0.03610666592915853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,7,0.04558933277924856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,7,0.036159999668598175
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,15,0.04533333579699198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,15,0.03621866554021835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,1,0.09336533149083455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,31,0.04593066871166229
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,31,0.03631466627120972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,1,0.05635199944178263
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,63,0.046623999873797096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,63,0.03673599908749262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,3,0.09434666236241658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,3,0.05644799768924713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,7,0.09635200103123982
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,7,0.05646933118502299
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,15,0.09822400410970052
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,15,0.05641066531340281
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,31,0.11148800452550252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,31,0.057018667459487915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,63,0.17844265699386597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,63,0.057589332262674965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,1,0.14450666308403015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,1,0.09803199768066406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,3,0.09783466657002766
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,3,0.14691733320554098
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,7,0.1511786679426829
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,15,0.16703999042510986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,7,0.09829866886138916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,15,0.09834133585294087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,31,0.19957866271336874
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,31,0.09862400094668071
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,1,0.2476159930229187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,63,0.31505600611368817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,63,0.10633066296577454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,1,0.17690666516621908
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,3,0.2569813330968221
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,3,0.17733333508173624
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,7,0.2765386700630188
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,7,0.17697600523630777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,15,0.2998560070991516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,31,0.3357706864674886
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,15,0.1771893302599589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,31,0.18296533823013306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,63,0.5265599886576334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,63,0.2777866721153259
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,1,0.12110400199890137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,1,0.06306666632493337
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,7,0.12562666336695352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,3,0.12135466933250427
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,3,0.06303999821345012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,15,0.1454026699066162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,15,0.0631573349237442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,7,0.06316266457239787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,31,0.17870932817459106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,31,0.06322133541107178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,1,0.10331199566523235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,3,0.27593066294987995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,3,0.10339732964833577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,1,0.2653759916623433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,7,0.293013334274292
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,15,0.28893333673477173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,7,0.10353599985440572
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,15,0.10372799634933472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,31,0.10532266894976298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,1,0.38068799177805585
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,1,0.1895093321800232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,3,0.3919626474380493
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,3,0.18974934021631876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,7,0.39410666624705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,7,0.19049066305160522
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,15,0.4096906582514445
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,31,0.33632532755533856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,15,0.19526932636896768
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,31,0.465776006380717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,1,0.5917919874191284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,31,0.21594132979710898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,1,0.34677334626515705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,3,0.6328800121943156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1,0.009493333597977957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,1,0.011861333002646765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,3,0.009392000113924345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,3,0.34889066219329834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,3,0.011802667131026586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,7,0.6243733167648315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,7,0.009530666594703993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,15,0.00943999985853831
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,31,0.009632000078757605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,31,0.01184533288081487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,7,0.011866666376590729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,63,0.009621333330869675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,63,0.011834666132926941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,127,0.009930666536092758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,127,0.011978667229413986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,255,0.013679999858140945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,15,0.01198400060335795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,7,0.35450132687886554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,255,0.013546666751305262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,511,0.014266667266686758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,511,0.014981333166360855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1023,0.016037333756685257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,1023,0.018063999712467194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,2047,0.022848000129063923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,2047,0.02383466561635335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,4095,0.028570666909217834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,4095,0.03595733394225439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,8191,0.03797333439191183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,8191,0.059605335195859276
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,3,0.01301866645614306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1,0.012981332838535309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,1,0.011920000116030375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,3,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,7,0.011882666498422623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,7,0.01302933320403099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,15,0.013125333935022354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,15,0.01179733375708262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,31,0.013237333546082178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,31,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,63,0.013210666676362356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,63,0.011855999628702799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,127,0.01357866699496905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,127,0.012069333344697952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,255,0.016063999384641647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,255,0.013647999614477158
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,511,0.017018667111794155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,511,0.015119999647140503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1023,0.022890667120615642
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,1023,0.018229333062966663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,2047,0.028160000840822857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,2047,0.02402666707833608
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,4095,0.04182399809360504
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,4095,0.03623466690381368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,15,0.6440426508585612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,8191,0.20196266969045004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,8191,0.059861332178115845
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1,0.011936000237862269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1,0.013093333691358566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,3,0.01314666618903478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,7,0.013077333569526672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,3,0.011861333002646765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,15,0.013157332936922709
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,15,0.012085333466529846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,31,0.013194666554530462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,31,0.01184533288081487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,63,0.013370666652917862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,63,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,127,0.01379199946920077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,7,0.01211200033624967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,255,0.016538667182127636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,127,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,255,0.014111999422311783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,511,0.0216799999276797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,511,0.01570133368174235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,2047,0.036559998989105225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1023,0.026933332284291584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,2047,0.025098666548728943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1023,0.018581333259741466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,4095,0.20478399594624838
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,4095,0.03754666695992152
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,8191,0.19510932763417563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1,0.013946666071812311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,8191,0.393887996673584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,3,0.013258667041858038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,15,0.3601066668828328
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,3,0.013909333695967993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,7,0.013253333667914072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,7,0.013951999445756277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1,0.012991999586423239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,31,0.013397333522637686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,15,0.013925333817799887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,15,0.01333333303531011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,63,0.013487999637921652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,63,0.014197333405415217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,127,0.014741333822409311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,127,0.014240000396966934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,31,0.7342133522033691
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,255,0.021205333371957142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,255,0.01709866647919019
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,511,0.02641066660483678
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1023,0.035818666219711304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,511,0.019541333119074505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,31,0.014159999787807465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1023,0.024842667082945507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,4095,0.3898186683654785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,31,0.38422401746114093
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,2047,0.0352906659245491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,4095,0.1970026691754659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,8191,0.37564265727996826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,8191,0.7589653333028158
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1,0.009285333255926767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,1,0.01219733307758967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,3,0.009466666728258133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,3,0.012122667084137598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,7,0.00943999985853831
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,2047,0.2026453415552775
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,15,0.009445333232482275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,15,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,31,0.009530666594703993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,31,0.012058666596810022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,63,0.012154666086037954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,63,0.009722666814923286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,127,0.009765333185593287
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,127,0.012458667159080505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,255,0.014053333550691605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,255,0.013973332941532135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,511,0.015253332753976187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,511,0.016927999754746754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1023,0.021386665602525074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,1023,0.0229120006163915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,2047,0.02661866694688797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,2047,0.03488533447186152
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1,0.013130666067202887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,1,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,7,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,4095,0.05877333382765452
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,3,0.013306666165590286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,3,0.012229333321253458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,7,0.013274667163689932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,7,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,4095,0.04018666595220566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,15,0.013424000392357508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,31,0.013359999905029932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,31,0.012165332833925882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,63,0.013471999516089758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,15,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,127,0.014085333794355392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,63,0.012613333761692047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,255,0.01708799973130226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,255,0.014192000031471252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,127,0.012565333396196365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,511,0.022495999932289124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,511,0.017221332838137943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1023,0.027637332677841187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,1023,0.02325333406527837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,4095,0.2006453275680542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,4095,0.05902933577696482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1,0.013359999905029932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,2047,0.035162667433420815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,2047,0.03742400060097376
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,3,0.013621332744757334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,7,0.013546666751305262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,15,0.01349866638580958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,3,0.012319999436537424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,7,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,15,0.012448000411192576
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,63,0.013989333063364029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,31,0.013674666484196981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,127,0.014373333503802618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,63,0.012730666746695837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,127,0.013093333691358566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,31,0.012458667159080505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,255,0.014720000326633453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,255,0.02202133337656657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,2047,0.204367995262146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,511,0.026895999908447266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,511,0.017829333742459614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,2047,0.036415999134381614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,4095,0.3924959897994995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1023,0.036506667733192444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1023,0.024186665813128155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1,0.017045332739750545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,4095,0.19413334131240845
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1,0.014538666854302088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,3,0.014490666488806406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,7,0.017423999806245167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,15,0.014511999984582266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,31,0.017397332936525345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,3,0.01718933383623759
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,7,0.01444799949725469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,31,0.014592000593741735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,63,0.015087999403476715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,15,0.0173333336909612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,63,0.01788266624013583
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,127,0.015919999529918034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,127,0.01889066646496455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,255,0.023605334262053173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,255,0.018757333358128864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1023,0.2198986609776815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,511,0.03366933266321818
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,511,0.02382933348417282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1023,0.03395200024048487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,2047,0.3881973425547282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,2047,0.19672532876332602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,3,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,3,0.010698666175206503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,7,0.00884799969693025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1,0.00874133345981439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,4095,0.3747040033340454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,1,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,7,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,15,0.008853333070874214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,31,0.008922666932145754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,15,0.010698666175206503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,4095,0.7562613487243652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,63,0.009002666920423508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,31,0.010768000036478043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,63,0.010746666540702185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,127,0.00921066664159298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,127,0.010773333410422007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,255,0.012629333883523941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,511,0.012896000097195307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,255,0.011727999895811081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1023,0.013679999858140945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,511,0.011781333635250727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,2047,0.014720000326633453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,4095,0.016917333006858826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,1023,0.011551999797423681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,2047,0.011749333391586939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,8191,0.021615999440352123
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,16383,0.031370667119820915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,4095,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,8191,0.013770667215188345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1,0.008767999708652496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,16383,0.016805333395799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,3,0.008858666444818178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,1,0.010703999549150467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,3,0.010687999427318573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,15,0.00884799969693025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,7,0.008863999818762144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,15,0.010821333775917688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,7,0.010869332899649939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,31,0.008874666566650072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,31,0.010725333044926325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,63,0.00877333308259646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,63,0.010869332899649939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,255,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,127,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,127,0.00919999989370505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,255,0.011695999652147293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,511,0.012650666137536367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,511,0.011535999675591787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1023,0.013327999661366144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,1023,0.011839999506870905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,2047,0.014426667243242264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,4095,0.016917333006858826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,2047,0.012122667084137598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,4095,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,8191,0.021829334398110706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,8191,0.013946666071812311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,16383,0.0354666660229365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1,0.008816000074148178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,16383,0.017050666113694508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,3,0.00874133345981439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,3,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,7,0.00884799969693025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,15,0.008922666932145754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,7,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,31,0.00878399983048439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,15,0.010885333021481832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,31,0.010938666760921478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,63,0.00898133342464765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,63,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,127,0.00919999989370505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,255,0.012357333054145178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,127,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,511,0.012709333250919977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,255,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,511,0.011866666376590729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1023,0.013370666652917862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,2047,0.014607999473810196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1023,0.012080000092585882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,4095,0.017445333302021027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,2047,0.012367999802033106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,4095,0.01301866645614306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,8191,0.026352000733216602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,16383,0.031386665999889374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,8191,0.014677333335081736
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,16383,0.017557332913080852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,3,0.008799999952316284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,7,0.008869333192706108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1,0.00877333308259646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,3,0.011264000087976456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,15,0.008842666943868002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,31,0.008837333569924036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,15,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,7,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,127,0.00915733352303505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,31,0.011194666226704916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,63,0.009008000294367472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,255,0.012367999802033106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,511,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,63,0.011802667131026586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,2047,0.015194666882356008
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,511,0.014058666924635569
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1023,0.014271999398867289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,255,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1023,0.013631999492645264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,2047,0.014682666709025701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,4095,0.021941334009170532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,8191,0.027098665634791057
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,4095,0.015717333803574245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,8191,0.018522666146357853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1,0.00873066671192646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,16383,0.023525332411130268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,16383,0.036720000207424164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,1,0.011050666371981302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,3,0.009045333291093508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,7,0.008949333180983862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,7,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,31,0.01099733387430509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,63,0.00921066664159298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,15,0.009018666421373686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,63,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,127,0.009082666908701261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,31,0.00914666677514712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,127,0.011168000598748526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,255,0.012613333761692047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,255,0.011855999628702799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,511,0.011834666132926941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,511,0.012986666212479273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1023,0.013503999759753546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,1023,0.011930666863918304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,2047,0.014773332824309668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,4095,0.01729600007335345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,2047,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,4095,0.013690666606028875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,8191,0.01664000004529953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,8191,0.022485333184401195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,16383,0.022677332162857056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,16383,0.03668266783157984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1,0.012074666718641916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,3,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,3,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,7,0.01210133358836174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,7,0.010965333630641302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,31,0.012330666184425354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,15,0.01102399950226148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,63,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,31,0.010885333021481832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,63,0.010949333508809408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,127,0.012661332885424295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,255,0.015024000157912573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,15,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,127,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,511,0.015237333873907724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,255,0.011690666278203329
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1023,0.015909332782030106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,511,0.014853333433469137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,1023,0.012058666596810022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,2047,0.012293333808581034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,4095,0.019866666446129482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,2047,0.017194667210181553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,4095,0.013845333208640417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,8191,0.016800000021855038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,8191,0.02951466788848241
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,16383,0.03460799902677536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,16383,0.023242667317390442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1,0.012026666353146235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,3,0.012272000312805176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,7,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1,0.011087999989589056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,3,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,7,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,15,0.012330666184425354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,15,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,31,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,31,0.01110400011142095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,127,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,255,0.014794666320085526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,63,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,255,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,511,0.01523200049996376
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,511,0.012058666596810022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1023,0.016016000260909397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,4095,0.02475733309984207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1023,0.012309333930412928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,2047,0.012879999975363413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,2047,0.017632000148296356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,4095,0.014346666634082794
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,8191,0.029663999875386555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,8191,0.017498667041460674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,16383,0.02367466688156128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,16383,0.03928533444801966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,3,0.012250666817029318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,3,0.01145600030819575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,7,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,7,0.011418666690587997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,15,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,15,0.012250666817029318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,31,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,63,0.011359999577204386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,63,0.01246400053302447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,31,0.011498666057984034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,127,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,255,0.01482133318980535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,511,0.015413332730531693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,127,0.012613333761692047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1023,0.016549333930015564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,255,0.014138666292031607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1023,0.014479999740918478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,511,0.01423466702302297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,2047,0.01573866605758667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,4095,0.02731200059254964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,8191,0.03701333453257879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,2047,0.02213866760333379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,4095,0.01841066653529803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,8191,0.023333333432674408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,16383,0.2230293353398641
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,16383,0.0335359995563825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,1,0.012837332983811697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1,0.009829333052039146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,3,0.012949333836634954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,3,0.009839999799927076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,7,0.00973866693675518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,7,0.012869333227475485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,15,0.009872000043590864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,15,0.012885333349307379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,31,0.012938667088747025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,63,0.010159999753038088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,31,0.0100426667680343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,63,0.012842666357755661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,127,0.010501333822806677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,127,0.013199999928474426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,255,0.016106666376193363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,255,0.015135999768972397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,511,0.017946666727463405
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,511,0.021914665897687275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1023,0.026869334280490875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,1023,0.023951999843120575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1,0.01414399966597557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,2047,0.03697066754102707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,1,0.013082666943470636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,3,0.01421333352724711
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,2047,0.035877334574858345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,3,0.012991999586423239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,7,0.014352000008026758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,7,0.013045333325862885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,15,0.013178666432698568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,31,0.014378666877746582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,15,0.014208000153303146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,63,0.013269333789745966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,31,0.01310933381319046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,127,0.015317333241303762
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,63,0.014815999815861383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,127,0.013807999591032663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,255,0.01533866673707962
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,255,0.02363733450571696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,511,0.018383999665578205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,511,0.028773332635561626
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1023,0.03861333429813385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,1023,0.024698667228221893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1,0.0176959993938605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1,0.015130666395028433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,3,0.017653333644072216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,3,0.015098666151364645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,2047,0.20374933878580728
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,2047,0.037104000647862755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,15,0.01798933371901512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,15,0.015210667004187902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,7,0.018112000077962875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,31,0.017978666971127193
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,7,0.015365333606799444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,31,0.015216000378131866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,127,0.019600000232458115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,127,0.01670933390657107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,255,0.024725332856178284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,255,0.019146667172511418
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,63,0.01565333331624667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,511,0.03458133339881897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,511,0.02442666639884313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,63,0.018543999642133713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,2047,0.39214932918548584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,2047,0.20015466213226318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1,0.02495466669400533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1023,0.20307733615239462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1023,0.03458133339881897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1,0.02109866589307785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,3,0.025274666647116344
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,3,0.020986666282018025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,7,0.02550400048494339
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,7,0.021141332884629566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,31,0.02128533273935318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,31,0.02571733295917511
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,63,0.02170666555563609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,63,0.026005332668622334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,15,0.025466665625572205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,15,0.021146667500336964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,127,0.02789866675933202
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,127,0.022970666488011677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,255,0.028351999819278717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1023,0.3938719828923543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,255,0.03786666691303253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1023,0.198527991771698
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,511,0.03775999943415324
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1,0.008949333180983862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,3,0.008943999807039896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,2047,0.7626667022705078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,511,0.2039626638094584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,1,0.011168000598748526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,7,0.011141333729028702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,7,0.008992000172535578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,15,0.01109333336353302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,15,0.009136000027259191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,2047,0.3792426586151123
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,63,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,31,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,63,0.009173333023985228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,127,0.009434666484594345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,127,0.011317333827416102
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,255,0.012858666479587555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,255,0.012047999848922094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,511,0.013141332815090815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,511,0.012015999605258306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,3,0.011007999380429586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,31,0.009029333169261614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,1023,0.012202666451533636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1023,0.013728000223636627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,2047,0.013754667093356451
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,4095,0.018218666315078735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,8191,0.027722666660944622
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,4095,0.01670933390657107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,8191,0.022613334159056347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,2047,0.01533866673707962
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,16383,0.03311999887228012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1,0.012319999436537424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,16383,0.03469333300987879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,3,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,1,0.011264000087976456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,7,0.012453333785136541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,3,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,15,0.012608000387748083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,15,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,7,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,63,0.012709333250919977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,31,0.012538666526476542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,31,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,63,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,127,0.012784000486135483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,255,0.015135999768972397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,127,0.011407999942700068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,255,0.012458667159080505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,511,0.015482666591803232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1023,0.016384000579516094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,511,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,1023,0.012309333930412928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,2047,0.017952000101407368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,4095,0.02534399926662445
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,2047,0.013877333452304205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,8191,0.030234667162100475
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,4095,0.01703466723362605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,8191,0.022842665513356526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,16383,0.040037333965301514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,16383,0.03491200009981791
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1,0.012426666915416718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,3,0.012517333030700684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,3,0.011407999942700068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,15,0.012586666891972223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,7,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,31,0.012661332885424295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,15,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,7,0.01250133290886879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,63,0.013167999684810638
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,31,0.011317333827416102
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,127,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,63,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,127,0.011434666812419891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,255,0.015226667126019796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,511,0.015685333559910457
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,255,0.012330666184425354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1023,0.016735999534527462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,511,0.012517333030700684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,2047,0.022490667800108593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1023,0.012762666990359625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,2047,0.014469332993030548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,4095,0.027744000156720478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,4095,0.017407999684413273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,8191,0.037434667348861694
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,8191,0.023984000086784363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1,0.012448000411192576
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,16383,0.03615466753641764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,16383,0.2034613291422526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1,0.013221333424250284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,3,0.013232000172138214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,15,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,31,0.01249066616098086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,7,0.013242666920026144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,7,0.012655999511480331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,15,0.013034666577974955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,31,0.013237333546082178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,127,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,63,0.012869333227475485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,63,0.013370666652917862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,127,0.013728000223636627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,255,0.014303999642531076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,511,0.016085332880417507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,255,0.015530666957298914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,2047,0.02664000044266383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1023,0.015487999965747198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,511,0.014720000326633453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1023,0.021333334346612293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,4095,0.04048000027736028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,2047,0.018351999421914417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,4095,0.023562667270501454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,8191,0.20217067003250122
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,8191,0.033701332906881966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,16383,0.3889760176340739
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,16383,0.20805867513020834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,1,0.014282666146755219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1,0.01033599985142549
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,3,0.0143306665122509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,3,0.010426666587591171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,7,0.010277333358923594
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,7,0.014229333649079004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,15,0.010314666976531347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,15,0.014335999886194864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,31,0.014357333381970724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,31,0.010458666831254959
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,63,0.014495999862750372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,63,0.010640000303586325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,127,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,127,0.015061333775520325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,255,0.02422933280467987
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,255,0.016528000434239704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,511,0.019551999866962433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,511,0.029109333952267964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1,0.018933333456516266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,1023,0.025648000339667004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1023,0.03882666677236557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,1,0.016309333344300587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,3,0.016352000335852306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,3,0.019050666441520054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,15,0.019248000035683315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,7,0.01623999948302905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,7,0.01923199991385142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,15,0.016250666230916977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,31,0.01934933289885521
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,31,0.016336000214020412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,63,0.01979200045267741
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,63,0.01691199963291486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,127,0.020997333029905956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,127,0.017887999614079792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,255,0.0258240004380544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,255,0.020341333001852036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,511,0.03566933423280716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,511,0.025642665723959606
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1,0.025978667040665943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1,0.02235200007756551
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,3,0.026154667139053345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1023,0.20069867372512817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,1023,0.035887998839219414
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,3,0.022218666970729828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,15,0.026517334083716076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,15,0.02248000105222066
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,7,0.026394667724768322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,7,0.022522665560245514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,31,0.026634665826956432
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,63,0.022821334501107533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,127,0.028837333122889202
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,127,0.024522667129834492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,31,0.02256533255179723
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,63,0.02720533311367035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,255,0.029477333029111225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,255,0.03878399978081385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1,0.03963200002908707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1,0.033813332517941795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,511,0.20863999923070273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,511,0.03899733225504557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,3,0.033770665526390076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,3,0.04028266668319702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1023,0.40035200119018555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1023,0.20329066117604574
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,7,0.04055466751257578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,15,0.04049066702524821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,15,0.033904001116752625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,31,0.041162667175134025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,7,0.033743999898433685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,63,0.034261333445707955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,127,0.043893332282702126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,127,0.03489600121974945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,63,0.041690667470296226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,31,0.03402133285999298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,255,0.2092319925626119
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,255,0.045610666275024414
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1023,0.7759626706441244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1023,0.38600532213846844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,511,0.20272533098856607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,511,0.4018933375676473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,1,0.014826666563749313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,3,0.018911999960740406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,7,0.015061333775520325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,15,0.018885333091020584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,1,0.018986667195955913
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,31,0.015450666348139444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,3,0.015013333410024643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,31,0.01893866683046023
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,15,0.015200000256299973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,63,0.015642666568358738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,7,0.018858666221300762
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,63,0.019509332875410717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,127,0.01703466723362605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,127,0.020245333512624104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,255,0.021967999637126923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,1,0.02921066681543986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,255,0.02290133386850357
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,1,0.02495466669400533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,3,0.029103999336560566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,3,0.02496533344189326
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,7,0.02951466788848241
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,7,0.02480533222357432
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,15,0.029706666866938274
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,31,0.025029333929220837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,15,0.025040000677108765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,31,0.029909332593282063
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,63,0.030266667405764263
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,63,0.026677332818508148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,127,0.03217600037654241
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,127,0.027093333502610523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,255,0.04223999877770742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,1,0.04288533329963684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,1,0.03673599908749262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,255,0.03230399886767069
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,3,0.04330666859944662
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,3,0.03677333394686381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,7,0.043978666265805565
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,15,0.04410133262475332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,15,0.036746665835380554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,31,0.04465599854787191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,7,0.036720000207424164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,31,0.03678400069475174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,63,0.045034666856129967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,63,0.037317333122094475
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,127,0.04744533201058706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,1,0.0673280010620753
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,1,0.05604266623655955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,3,0.0684799998998642
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,127,0.038373333712418876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,255,0.21806933482487997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,255,0.04921066761016846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,3,0.05629333357016245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,15,0.06940799951553345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,7,0.05606933434804281
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,7,0.06968533496061961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,15,0.056101332108179726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,63,0.057189335425694786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,31,0.07022933165232341
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,31,0.05716800192991892
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,127,0.21838400761286417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,127,0.059952000776926674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,255,0.2128373384475708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,255,0.4150666793187459
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,1,0.0239680012067159
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,1,0.030368000268936157
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,3,0.024266667664051056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,7,0.024421334266662598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,15,0.0303413321574529
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,31,0.02499199906984965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,3,0.030159999926884968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,31,0.030181333422660828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,7,0.03044266750415166
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,15,0.02442666639884313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,63,0.025216000775496166
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,63,0.031093334158261616
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,127,0.027749332288901012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,127,0.032255999743938446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,1,0.04855999847253164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,1,0.0421066681543986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,3,0.0490880012512207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,3,0.04205866654713949
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,7,0.049685334165891014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,7,0.042133331298828125
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,15,0.042303999265034996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,15,0.049551998575528465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,31,0.05045333504676819
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,31,0.0423573354880015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,63,0.05129600067933401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,63,0.04276266694068909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,1,0.07387733459472656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,127,0.05486399928728739
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,127,0.04387199878692627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,1,0.06329066554705302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,3,0.06313066681226094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,3,0.07540266712506612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,7,0.06323733429114024
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,15,0.07640000184377034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,7,0.07657599945863088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,15,0.063509335120519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,31,0.0769706666469574
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,63,0.07971199850241344
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,31,0.06368533273537953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,63,0.0639466643333435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,1,0.12454400459925334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,127,0.23929067452748617
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,1,0.10265066226323445
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,127,0.06706133484840393
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,3,0.12829333543777466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,3,0.1027733286221822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,7,0.1283093293507894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,7,0.10286933183670044
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,15,0.1030346651871999
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,15,0.12876266241073608
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,31,0.13150933384895325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,31,0.10337600111961365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,63,0.23991467555363974
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1,0.009232000137368837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,7,0.00927466650803884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,7,0.011354666203260422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,15,0.009098666409651438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,3,0.00927466650803884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,3,0.011194666226704916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,1,0.011402666568756104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,15,0.011482667177915573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,31,0.009402666861812273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,31,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,127,0.009685333197315535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,63,0.011296000331640244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,63,0.009413333609700203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,255,0.012986666212479273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,255,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,511,0.013338666409254074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,511,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1023,0.014149333039919535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,63,0.10418666402498881
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,1023,0.014032000054915747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,2047,0.016186666985352833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,2047,0.017008000363906223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,4095,0.023423999547958374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,4095,0.023077333966890972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,8191,0.03489066660404205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,16383,0.05874133110046387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1,0.012586666891972223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,1,0.011402666568756104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,16383,0.038719999293486275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,3,0.012741333494583765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,7,0.012714666624863943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,3,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,15,0.012752000242471695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,7,0.011312000453472137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,15,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,31,0.012821332861979803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,31,0.011498666057984034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,63,0.012981332838535309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,63,0.01139733319481214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,127,0.013023999830087027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,127,0.011605333536863327
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,8191,0.028944000601768494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,127,0.44233067830403644
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,127,0.23613333702087402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,255,0.012485332787036896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,255,0.01543466622630755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,511,0.016016000260909397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,511,0.01267733300725619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1023,0.01706133286158244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,1023,0.014181333283583323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,4095,0.02834133307139079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,2047,0.01716800034046173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,8191,0.03789866715669632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,8191,0.035071998834609985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,16383,0.19993066787719727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,16383,0.05945600072542826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1,0.011306667079528173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,3,0.012757333616415659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1,0.012661332885424295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,3,0.011488000551859537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,7,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,15,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,15,0.01146666705608368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,31,0.012746666868527731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,31,0.011472000430027643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,63,0.012906666845083237
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,63,0.011535999675591787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,127,0.011930666863918304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,2047,0.022976001103719074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,127,0.01332266628742218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,4095,0.02346666653951009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,255,0.015546667079130808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,255,0.012746666868527731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,511,0.016575999557971954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,511,0.013242666920026144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1023,0.021615999440352123
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,2047,0.026741333305835724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,2047,0.017802666872739792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,4095,0.04071466624736786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1023,0.015008000036080679
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,4095,0.024149333437283833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,8191,0.20245333512624106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,8191,0.0364479993780454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,16383,0.39301331837972003
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,16383,0.192522664864858
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1,0.01339200014869372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1,0.012847999731699625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,3,0.013370666652917862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,7,0.012778667112191519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,3,0.012746666868527731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,15,0.012847999731699625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,7,0.013482666263977686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,15,0.013573333621025085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,31,0.012784000486135483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,31,0.013338666409254074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,63,0.01302933320403099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,63,0.013503999759753546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,127,0.013455999394257864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,127,0.014111999422311783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,255,0.016208000481128693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,255,0.015082667271296183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,511,0.015909332782030106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,511,0.021029333273569744
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1023,0.02590399980545044
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,2047,0.023893333971500397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1023,0.018842666099468868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,2047,0.035818666219711304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,4095,0.2018293341000875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,4095,0.03373866776625315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,8191,0.42534931500752765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,8191,0.1949333349863688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,16383,0.37172798315684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,16383,0.7558720111846924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,1,0.04192000130812327
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,1,0.034703999757766724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,3,0.034645333886146545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,3,0.042506664991378784
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,7,0.0429066667954127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,7,0.03488533447186152
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,15,0.04252799848715464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,15,0.03458133339881897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,31,0.04342933495839437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,31,0.03478399912516276
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,63,0.03496533383925756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,63,0.04422399898370107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,1,0.08524800340334575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,1,0.05471999943256378
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,3,0.08611733714739482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,7,0.08699199557304382
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,3,0.05513600011666616
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,7,0.05500266452630361
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,15,0.08736532926559448
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,15,0.05509866774082184
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,31,0.08950933814048767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,1,0.1369599997997284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,31,0.05541866521040598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,63,0.13804800311724344
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,1,0.09616532921791077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,63,0.056602666775385536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,3,0.13904000322024027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,3,0.09603733817736308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,7,0.14106667041778564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,7,0.09636267026265462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,15,0.14323733250300089
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,15,0.09659733374913533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,31,0.1702293356259664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,31,0.09685333569844563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,63,0.30558399359385174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,1,0.23975465695063272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,63,0.10116799672444661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,1,0.17626667022705078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,3,0.24414400259653726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,3,0.17515732844670615
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,7,0.250165323416392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,7,0.17523733774820963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,15,0.2760586738586426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,15,0.17575999101003012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,31,0.32441065708796185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,31,0.1788853406906128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,63,0.49324798583984375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,63,0.24352532625198364
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,1,0.07462400197982788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,1,0.05974400043487549
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,3,0.07573866844177246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,3,0.05991466840108236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,7,0.0776693324247996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,7,0.059802666306495667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,15,0.08138133088747661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,15,0.060362666845321655
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,31,0.10884267091751099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,31,0.0602453351020813
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,1,0.20518932739893594
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,1,0.10014933347702026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,3,0.21660266319910684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,3,0.10003200173377991
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,7,0.23490132888158163
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,7,0.10039466619491577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,15,0.23770666122436523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,15,0.10065066814422607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,31,0.2659200032552083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,31,0.1011946698029836
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,1,0.3105439941088359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,1,0.185754656791687
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,3,0.3481706778208415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,3,0.1857866644859314
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,7,0.3475840091705322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,7,0.1860426664352417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,15,0.35523732503255206
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,15,0.1867146690686544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,31,0.42585599422454834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,1,0.5501866738001505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,31,0.2002346714337667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,1,0.3434026638666789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,3,0.5823786656061808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,7,0.5791733264923096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,3,0.34359467029571533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,1,0.011648000528415045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,3,0.009296000003814697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,7,0.34598398208618164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1,0.009237333511312803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,3,0.011648000528415045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,7,0.009317333499590555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,15,0.009306666751702627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,15,0.011519999553759893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,31,0.009408000235756239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,31,0.011493333925803503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,63,0.009578666960199675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,63,0.011477333803971609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,7,0.011722666521867117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,127,0.009717333440979322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,127,0.011653333902359009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,255,0.013178666432698568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,255,0.01313599944114685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,511,0.013866666704416275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1023,0.015253332753976187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,511,0.01444799949725469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,1023,0.017530667285124462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,2047,0.0220320001244545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,2047,0.023711999257405598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,4095,0.026901334524154663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,4095,0.0355679988861084
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,8191,0.03688533355792364
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,8191,0.05923733115196228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1,0.012906666845083237
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,3,0.012842666357755661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,1,0.011621333658695221
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,3,0.011674666156371435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,7,0.012896000097195307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,7,0.011557333171367645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,15,0.01303999995191892
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,15,0.01163200040658315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,31,0.013178666432698568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,31,0.011695999652147293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,63,0.01302933320403099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,63,0.011648000528415045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,127,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,127,0.011941333611806234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,255,0.015861333658297855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,255,0.013338666409254074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,511,0.016719999412695568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,511,0.014741333822409311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1023,0.022367998957633972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,1023,0.017818666994571686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,2047,0.024106666445732117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,4095,0.041365332901477814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,2047,0.027632000545660656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,15,0.353216012318929
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,15,0.5808480183283488
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,8191,0.19970667362213135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,4095,0.0359199990828832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,8191,0.05961066484451294
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1,0.012805332740147909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1,0.011920000116030375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,7,0.011855999628702799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,15,0.013130666067202887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,3,0.011898666620254517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,31,0.012949333836634954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,15,0.011706666400035223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,31,0.01190399999419848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,63,0.013183999806642532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,63,0.011781333635250727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,3,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,127,0.013733333597580591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,127,0.012272000312805176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,255,0.016303999970356624
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,255,0.013754667093356451
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,511,0.021498667697111767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,511,0.015381333728631338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1023,0.02664000044266383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,2047,0.03629866739114126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1023,0.01833600054184596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,2047,0.024533333877722423
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,4095,0.03709333389997482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,4095,0.2031360069910685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,8191,0.39204267660776776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,8191,0.1939893364906311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1,0.0138026662170887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,3,0.013066666821638743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,3,0.013850666582584381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,7,0.013674666484196981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,7,0.013232000172138214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,31,0.6358826557795206
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,15,0.013199999928474426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,31,0.01320533330241839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,15,0.01379199946920077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,31,0.013914667069911957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,63,0.013376000026861826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,255,0.021194666624069214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,63,0.01393066719174385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,127,0.014032000054915747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,255,0.01648533344268799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,511,0.026191999514897663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,31,0.37055468559265137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1023,0.035829332967599235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,511,0.01913600042462349
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1023,0.024522667129834492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,2047,0.20165866613388062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,2047,0.034330666065216064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,4095,0.19576533635457358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,4095,0.3887253204981486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,127,0.014746667196353277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,8191,0.3740319808324178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1,0.009306666751702627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,1,0.012096000214417776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,3,0.012053333222866058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,3,0.009359999870260557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,7,0.009413333609700203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,7,0.01191466674208641
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,15,0.00949866697192192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,15,0.012096000214417776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,31,0.009445333232482275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,31,0.012223999947309494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,8191,0.755786657333374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,63,0.009568000212311745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,127,0.009904000287254652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,63,0.012053333222866058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,127,0.012122667084137598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,255,0.014127999544143677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,255,0.013728000223636627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,511,0.014906667172908783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,511,0.016837333639462788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1023,0.021007999777793884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,1023,0.022629333039124806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,2047,0.02603733291228612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,2047,0.03454400102297465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1,0.013306666165590286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,1,0.011962667107582092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,4095,0.03966933240493139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,4095,0.058703998724619545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,3,0.013343999783198038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,7,0.013552000125249227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,3,0.012005332857370377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,7,0.01191466674208641
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,15,0.013194666554530462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,31,0.012223999947309494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,63,0.01351999988158544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,15,0.011999999483426413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,31,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,63,0.012229333321253458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,127,0.013834666460752487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,127,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,255,0.01703466723362605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,255,0.014042666802803675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,511,0.022255999346574146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,1023,0.022991999983787537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,511,0.01708799973130226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1023,0.02739733209212621
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,4095,0.19974400599797568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,4095,0.059061333537101746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,2047,0.03682666768630346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,2047,0.035162667433420815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1,0.013173333058754602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,7,0.01349866638580958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,3,0.012213333199421564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,15,0.012106666962305704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,31,0.013530666629473368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,31,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,3,0.013461332768201828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,7,0.012144000579913458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,15,0.013525333255529404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,63,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,127,0.014469332993030548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,63,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,255,0.021530665457248688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,127,0.012821332861979803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,255,0.01479999969402949
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,511,0.026538667579491932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,511,0.017792000124851864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,2047,0.20372267564137778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,2047,0.036271999279658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1023,0.023813332120577495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1023,0.03632533301909765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,4095,0.3920266628265381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1,0.014368000129858652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,4095,0.19326933224995932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1,0.01681600014368693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,3,0.016879999389251072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,7,0.017125333348910015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,3,0.014389333625634512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,15,0.01732800031701724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,7,0.014405333747466406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,15,0.014293332894643148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,31,0.017221332838137943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,31,0.014293332894643148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,63,0.017770666629076004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,63,0.014975999792416891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,127,0.018965333700180054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,127,0.015722667177518208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,255,0.02364266663789749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,255,0.018432000031073887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1023,0.2003893256187439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,511,0.02382933348417282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1023,0.033573334415753685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,511,0.03386666625738144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,2047,0.38892801602681476
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,2047,0.1965706745783488
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1,0.008496000121037165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,4095,0.7539520263671875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,3,0.010501333822806677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,7,0.010794666906197866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,7,0.008767999708652496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,3,0.008693333094318708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,15,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,15,0.00867733359336853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,31,0.010677333921194077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,31,0.008816000074148178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,1,0.010661333799362183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,4095,0.3737066586812337
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,63,0.008943999807039896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,63,0.01055466632048289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,127,0.009103999783595404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,127,0.01080000028014183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,255,0.012538666526476542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,511,0.011589333415031433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,255,0.011557333171367645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,511,0.012805332740147909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,1023,0.01157333329319954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,2047,0.01451733335852623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1023,0.01350933313369751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,2047,0.011717333147923151
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,4095,0.012330666184425354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,8191,0.013690666606028875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,8191,0.021322667598724365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,4095,0.01693333312869072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,16383,0.03073066721359889
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,1,0.011007999380429586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,3,0.010741333166758219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,7,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,16383,0.01658133293191592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,3,0.008837333569924036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,7,0.008874666566650072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1,0.008949333180983862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,15,0.00873066671192646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,31,0.008810666700204214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,15,0.010735999792814255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,31,0.010672000547250112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,63,0.008912000184257826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,127,0.009098666409651438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,63,0.010879999647537867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,127,0.011087999989589056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,255,0.012272000312805176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,255,0.01156266654531161
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,511,0.012645332763592402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,511,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1023,0.01331199953953425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,1023,0.011781333635250727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,2047,0.014597332725922266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,4095,0.016714667280515034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,4095,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,2047,0.011968000481526056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,8191,0.021829334398110706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,8191,0.013909333695967993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,16383,0.03561066587766012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1,0.008746666833758354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,16383,0.016965333372354507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,3,0.008816000074148178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,3,0.010762666662534079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,7,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,31,0.008832000195980072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,15,0.010890666395425797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,7,0.008874666566650072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,63,0.008885333314538002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,31,0.010794666906197866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,127,0.009114666531483332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,63,0.010853332777818045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,127,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,255,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,511,0.012624000509579977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,255,0.011770666887362799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,511,0.011946666985750198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1023,0.013327999661366144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,2047,0.014677333335081736
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1023,0.011989332735538483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,4095,0.01752000053723653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,2047,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,8191,0.026394667724768322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,4095,0.013141332815090815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,16383,0.031258667508761086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,16383,0.017685333887736004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,8191,0.014688000082969666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,3,0.008901333436369896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1,0.008629333227872849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,7,0.008896000062425932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,3,0.011168000598748526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,7,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,15,0.008949333180983862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,31,0.008890666688481966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,63,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,31,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,127,0.009183999771873156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,15,0.011621333658695221
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,63,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,255,0.012426666915416718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,511,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,255,0.012565333396196365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,2047,0.015103999525308609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,511,0.013797332843144735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1023,0.014186666657527288
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1023,0.013605333864688873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,2047,0.014725333700577417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,4095,0.021882665654023487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,8191,0.02681066592534383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,4095,0.015770666301250458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,8191,0.018250666558742523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,16383,0.03677866607904434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,16383,0.023754666248957317
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1,0.008832000195980072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,1,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,3,0.008826666822036108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,7,0.00892800030608972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,7,0.010863999525705973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,15,0.00895999992887179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,31,0.008938666433095932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,31,0.010853332777818045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,15,0.010693332801262537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,63,0.009152000149091085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,63,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,255,0.012613333761692047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,127,0.009301333377758661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,127,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,255,0.011861333002646765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,511,0.011839999506870905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,511,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,2047,0.014650666465361914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1023,0.013466666142145792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,1023,0.011999999483426413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,2047,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,8191,0.016575999557971954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,8191,0.022431999444961548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,4095,0.017130666722853977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,4095,0.013647999614477158
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,16383,0.036730666955312095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,16383,0.022543999056021374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1,0.01202133297920227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,3,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,7,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,7,0.012879999975363413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,3,0.010885333021481832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,1,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,15,0.01210133358836174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,15,0.010938666760921478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,31,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,63,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,31,0.011402666568756104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,127,0.012549333274364471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,255,0.01482133318980535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,127,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,255,0.011802667131026586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,511,0.015141333142916361
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,511,0.011909333368142446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1023,0.015791999797026317
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,2047,0.017279999951521557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,1023,0.011850666254758835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,2047,0.012453333785136541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,4095,0.019946667055288952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,4095,0.013776000589132309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,8191,0.029194665451844532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,8191,0.016741332908471424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,16383,0.034341332813103996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1,0.012085333466529846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,16383,0.022826666633288067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1,0.01108266661564509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,3,0.012144000579913458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,7,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,3,0.011007999380429586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,7,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,15,0.012309333930412928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,31,0.012293333808581034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,15,0.011039999624093374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,63,0.012650666137536367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,31,0.011503999431928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,127,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,63,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,255,0.015013333410024643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,127,0.011274666835864386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,255,0.01201066623131434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,511,0.015210667004187902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1023,0.016042667130629223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,511,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,2047,0.01756799966096878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1023,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,2047,0.012768000364303589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,4095,0.02477866659561793
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,8191,0.02972800036271413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,8191,0.01741333305835724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,16383,0.03924266745646795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,4095,0.01458666721979777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,16383,0.02359466751416524
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,3,0.012138667205969492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1,0.011551999797423681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,3,0.01137599969903628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,15,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,7,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,15,0.011530666301647821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,7,0.011482667177915573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,63,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,31,0.012400000045696894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,31,0.012069333344697952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,127,0.01259200026591619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,255,0.014874666929244995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,63,0.011402666568756104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,511,0.014357333381970724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1023,0.01666133354107539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,255,0.014170666535695394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,127,0.012080000092585882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,511,0.01563199982047081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1023,0.014538666854302088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,2047,0.022282667458057404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,15,0.009008000294367472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,4095,0.027189334233601887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,8191,0.03703466554482778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,8191,0.023445333043734234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,2047,0.015642666568358738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,16383,0.20231467485427856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,4095,0.018122666825850803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,1,0.012986666212479273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1,0.009626666704813639
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,3,0.009663999701539675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,3,0.012698666503032049
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,7,0.012703999876976013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,7,0.009818666925032934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,15,0.009770666559537252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,15,0.012768000364303589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,31,0.00980266680320104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,31,0.012762666990359625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,63,0.009989333028594652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,63,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,127,0.010549332946538925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,127,0.013237333546082178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,255,0.015557333827018738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,255,0.014592000593741735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,511,0.017738666385412216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,511,0.021482666333516438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1023,0.026421333352724712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,1023,0.02363733450571696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1,0.013893333574136099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,1,0.012874666601419449
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,2047,0.035904000202814736
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,2047,0.03565333286921183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,3,0.014021333307027817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,3,0.012826666235923767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,7,0.013936000565687815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,7,0.013007999708255133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,15,0.013738666971524557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,31,0.0141546664138635
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,15,0.012784000486135483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,31,0.01314666618903478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,63,0.014458666245142618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,63,0.01313599944114685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,127,0.014959999670584997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,127,0.013552000125249227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,255,0.015418666104475657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,255,0.0230880007147789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,511,0.02807466685771942
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,511,0.018330667167901993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1023,0.03793599953254064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,1023,0.02441066751877467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1,0.014901333798964819
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1,0.017642666896184284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,2047,0.036943999429543815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,2047,0.20176533857981363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,3,0.017685333887736004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,3,0.014959999670584997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,7,0.017845333864291508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,7,0.015066667149464289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,31,0.01801066721479098
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,31,0.014943999548753103
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,63,0.018218666315078735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,15,0.01783466711640358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,15,0.01492799942692121
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,63,0.01563199982047081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,127,0.019424000134070713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,127,0.016309333344300587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,255,0.024725332856178284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,255,0.019039999693632126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,511,0.03408000121514002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,511,0.024170666933059692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,2047,0.3917280038197835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1023,0.2029973268508911
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,2047,0.19882667064666748
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1023,0.03453333427508672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,3,0.02502399931351344
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1,0.020879998803138733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,16383,0.03491200009981791
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,7,0.02513066679239273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1,0.024698667228221893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,15,0.025290665527184803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,15,0.021018666525681812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,3,0.020970667401949566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,31,0.021082667013009388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,31,0.02552533398071925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,63,0.025973332424958546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,63,0.0215786670645078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,7,0.020981334149837494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,127,0.02288000037272771
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,127,0.027589333554108936
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1023,0.42928000291188556
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,255,0.03751999884843826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1023,0.19750932852427164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,255,0.02828799933195114
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,3,0.008901333436369896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,511,0.203439990679423
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,511,0.037765334049860634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,1,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,3,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,2047,0.7612106800079346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,7,0.011109333485364914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,7,0.009056000038981438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,31,0.009066666786869368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,31,0.011274666835864386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,63,0.009061333412925402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,63,0.011109333485364914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,127,0.009397333487868309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,127,0.01126933346192042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,255,0.012655999511480331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,2047,0.4018239974975586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,511,0.013189333180586496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,511,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1023,0.013722666849692663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,255,0.011898666620254517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,15,0.00914666677514712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,15,0.011109333485364914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,1023,0.012085333466529846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,2047,0.013605333864688873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,4095,0.01798933371901512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,4095,0.016741332908471424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,8191,0.027376001079877216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,8191,0.02272533377011617
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,16383,0.032746667663256325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,16383,0.03449599941571554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,7,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,1,0.011168000598748526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,3,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,2047,0.015200000256299973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,3,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,7,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,15,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,31,0.012602667013804117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,15,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,31,0.01118933285276095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,127,0.012709333250919977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,63,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,127,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,255,0.015210667004187902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,63,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,255,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,511,0.012240000069141388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1023,0.016208000481128693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,511,0.015429332852363586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,1023,0.012400000045696894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,2047,0.017866666118303936
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,4095,0.025029333929220837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,2047,0.013973332941532135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,4095,0.016927999754746754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,8191,0.029946667452653248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,8191,0.022848000129063923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,16383,0.03976000100374222
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1,0.012453333785136541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,16383,0.034917332231998444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1,0.011418666690587997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,3,0.012645332763592402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,7,0.01250133290886879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,3,0.011264000087976456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,15,0.012538666526476542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,7,0.01146666705608368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,15,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,31,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,31,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,63,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,63,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,127,0.01292266696691513
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,255,0.014901333798964819
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,127,0.011616000284751257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,255,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,511,0.01563199982047081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,511,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,2047,0.022255999346574146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1023,0.016757333030303318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,2047,0.0144213338692983
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1023,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,4095,0.02756800005833308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,4095,0.01759999990463257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,8191,0.03728000074625015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,8191,0.02370133250951767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,16383,0.2035199999809265
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,16383,0.0358240008354187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1,0.013221333424250284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,3,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,3,0.01303999995191892
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,15,0.012661332885424295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,7,0.01314666618903478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,7,0.012709333250919977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,15,0.013306666165590286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,31,0.01257066677014033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,31,0.013077333569526672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,63,0.012698666503032049
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,63,0.013237333546082178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,127,0.012837332983811697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,255,0.015487999965747198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,511,0.015925332903862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,127,0.013749333719412485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1023,0.021205333371957142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,255,0.014458666245142618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,511,0.014661333213249842
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,2047,0.026575999955336254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1023,0.015552000453074774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,4095,0.040474665661652885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,2047,0.01825599993268649
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,4095,0.02346133440732956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,8191,0.2020906607309977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,8191,0.03344533344109853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,16383,0.38920001188913983
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,16383,0.19400533040364584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,1,0.01381333296497663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1,0.010048000141978264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,3,0.014181333283583323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,3,0.01002133327225844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,7,0.01009599988659223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,7,0.01404800017674764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,15,0.010069333637754122
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,15,0.013797332843144735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,31,0.013962666193644205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,31,0.010431999961535135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,63,0.014069333672523499
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,63,0.010485333700974783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,127,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,127,0.014538666854302088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,255,0.022837333381175995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,255,0.016176000237464905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1,0.01844800015290578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,511,0.019120000302791595
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,511,0.028010666370391846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,1023,0.025616000096003216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1023,0.03772266705830892
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,1,0.016176000237464905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,3,0.01852799952030182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,3,0.016048000504573185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,7,0.01886933296918869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,7,0.01597333326935768
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,15,0.018895999838908512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,31,0.016261332978804905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,31,0.019088000059127808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,15,0.016106666376193363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,63,0.01915733392039935
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,63,0.016607999801635742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,127,0.020506666352351505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,127,0.017370666066805523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,255,0.025407999753952026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,255,0.020213333268960316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,511,0.035205334424972534
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,511,0.025370667378107708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1,0.025381334125995636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1023,0.19909866650899252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1,0.022143999735514324
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,1023,0.03537066777547201
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,3,0.02588266630967458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,7,0.021941334009170532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,7,0.026159999271233875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,3,0.022042666872342426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,15,0.02222399910291036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,15,0.026101333399613697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,31,0.02630399912595749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,31,0.022261333962281544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,63,0.026704000929991405
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,63,0.02257599929968516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,127,0.028543998797734577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,127,0.024122667809327442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,255,0.03827733298142751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,255,0.029125332832336426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1,0.03942399968703588
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,511,0.2071946660677592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,511,0.03864533454179764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1,0.0335413341720899
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,3,0.03997333347797394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1023,0.3983039855957031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1023,0.2018186648686727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,3,0.0336053321758906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,15,0.040522667268911995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,15,0.03346666693687439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,7,0.0406986673672994
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,7,0.033370666205883026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,63,0.04186666508515676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,63,0.03429866582155228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,127,0.04420266548792521
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,127,0.034645333886146545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,31,0.04098133246103922
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,31,0.033626665671666466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,255,0.2084160049756368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,255,0.04484266539414724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1023,0.38284265995025635
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1023,0.7727680206298828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,511,0.39957865079243976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,1,0.014501333236694336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,511,0.20124266544977823
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,1,0.018250666558742523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,3,0.014655999839305878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,7,0.014741333822409311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,7,0.01817600056529045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,3,0.01815466706951459
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,15,0.018325333793958027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,31,0.018191999445358913
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,63,0.015466666469971338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,63,0.018816000471512478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,127,0.016314666718244553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,31,0.01504533365368843
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,127,0.01956266661485036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,15,0.014831999937693277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,255,0.021477334201335907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,255,0.022272000710169475
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,1,0.028218666712443035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,1,0.024266667664051056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,3,0.028501334289709728
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,3,0.024319998919963837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,7,0.028629332780838013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,7,0.0244159996509552
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,15,0.028773332635561626
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,15,0.02436800052722295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,31,0.029157333076000214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,31,0.024319998919963837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,63,0.029472000896930695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,63,0.02493866781393687
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,127,0.031221332649389904
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,127,0.026746665438016255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,255,0.041482667128245033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,255,0.03145600110292435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,1,0.042133331298828125
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,1,0.03594133257865906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,3,0.04268266757329305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,3,0.036373332142829895
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,7,0.04322666426499685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,7,0.03606399893760681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,15,0.043466667334238686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,31,0.04377066592375437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,15,0.03612799942493439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,31,0.03603733330965042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,63,0.044480000933011375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,63,0.036576000352700554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,127,0.04665599763393402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,127,0.03728533287843069
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,255,0.2154560089111328
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,1,0.06671999891599019
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,255,0.04805333415667216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,1,0.05533866584300995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,3,0.055530667304992676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,3,0.06794666747252147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,7,0.06860266625881195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,7,0.05554133156935374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,15,0.055546666185061135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,15,0.06883200009663899
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,31,0.055829331278800964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,31,0.06931200126806895
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,63,0.07110933462778728
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,63,0.05608533322811127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,127,0.21559999386469522
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,127,0.057914664347966514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,255,0.41097601254781085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,255,0.22331732511520386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,1,0.023205332458019257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,1,0.028805332879225414
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,3,0.023503998915354412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,3,0.028757333755493164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,7,0.023738667368888855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,15,0.02882133424282074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,31,0.024127999941507976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,31,0.028981332977612812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,15,0.023872000475724537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,7,0.028938665986061096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,63,0.024432001014550526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,1,0.04689066608746847
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,63,0.029418667157491047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,127,0.0268053337931633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,127,0.031136001149813335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,1,0.040607998768488564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,3,0.0472320020198822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,3,0.04083733260631561
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,7,0.04774933556715647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,7,0.04051200052102407
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,15,0.04782933493455251
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,15,0.04062933226426443
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,31,0.04869333406289419
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,31,0.04072533299525579
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,63,0.04926399886608124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,63,0.041120000183582306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,127,0.05273599922657013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,127,0.04266666869322459
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,1,0.07237866520881653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,1,0.06182933350404104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,3,0.07369600236415863
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,3,0.06187200049559275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,7,0.07459733386834462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,7,0.06182933350404104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,15,0.07459733386834462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,15,0.06200533111890157
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,31,0.07539199789365132
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,31,0.06198399762312571
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,63,0.07678399980068207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,63,0.06262399752934773
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,127,0.23244265715281168
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,127,0.06474133332570393
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,1,0.12262933452924092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,1,0.10108799735705058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,3,0.125408003727595
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,3,0.1011893351872762
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,7,0.1267093320687612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,31,0.1285760005315145
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,7,0.10107200344403584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,15,0.1013866662979126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1,0.00901333304742972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,15,0.12653332948684692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,31,0.10190932949384053
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,1,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,7,0.011205332974592844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,3,0.01126933346192042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,3,0.00914666677514712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,15,0.009103999783595404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,7,0.009183999771873156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,15,0.011178666104873022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,31,0.009237333511312803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,31,0.011205332974592844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,63,0.009328000247478485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,63,0.011109333485364914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,127,0.009541333342591921
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,255,0.012853333105643591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,127,0.011402666568756104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,255,0.012165332833925882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,511,0.01328533391157786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,511,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,1023,0.013770667215188345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1023,0.014058666924635569
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,63,0.23335999250411987
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,2047,0.01589866727590561
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,2047,0.01691199963291486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,4095,0.022810667753219604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,4095,0.023381332556406658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,8191,0.03487999985615412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,16383,0.03826666623353958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,63,0.10237866640090942
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,1,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,16383,0.058506667613983154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,3,0.01268799975514412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,7,0.012815999488035837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,15,0.012725333372751871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,7,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1,0.012469333906968435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,15,0.011253333340088526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,31,0.012837332983811697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,31,0.011237333218256632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,63,0.012725333372751871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,63,0.011445333560307821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,127,0.4344319899876912
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,3,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,127,0.011626667032639185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,8191,0.028378665447235107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,127,0.01302933320403099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,255,0.015594666202863058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,127,0.22952000300089517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,511,0.012554666648308435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,511,0.015909332782030106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1023,0.017114666601022083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,1023,0.0141546664138635
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,2047,0.02295999974012375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,4095,0.02787200113137563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,4095,0.023221333821614582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,8191,0.03508266558249792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,8191,0.037962667644023895
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,16383,0.059024001161257424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,16383,0.19913599888483682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,3,0.012719999998807907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1,0.012671999633312225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,3,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,7,0.012773333738247553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,255,0.01257066677014033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,7,0.011642667154471079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,15,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,31,0.011514666179815928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,63,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,15,0.012778667112191519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,63,0.011493333925803503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,31,0.012703999876976013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,127,0.013210666676362356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,127,0.011770666887362799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,255,0.015594666202863058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,255,0.01259200026591619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,511,0.016522667060295742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,2047,0.01709866647919019
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,2047,0.026464000344276428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,511,0.013104000439246496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1023,0.021642667551835377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1023,0.014773332824309668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,2047,0.017680000513792038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,4095,0.040618665516376495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,4095,0.02385066697994868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,8191,0.20303465922673544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,8191,0.03634133438269297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,16383,0.19273600975672403
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,16383,0.39103468259175617
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1,0.012618667135636011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1,0.013429333766301474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,3,0.013317332913478216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,3,0.012698666503032049
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,7,0.012863999853531519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,7,0.013440000514189402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,31,0.012965332716703415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,15,0.013295999417702356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,15,0.012842666357755661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,31,0.013445333888133367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,127,0.013408000270525614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,63,0.01293333371480306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,63,0.013616000612576803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,255,0.01600533351302147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,511,0.020853333175182343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1023,0.025909334421157837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,127,0.014064000298579534
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,511,0.015935999651749928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,255,0.015114666273196539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1023,0.018485333770513535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,2047,0.03570133447647095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,4095,0.20070399840672812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,2047,0.023775999744733173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,4095,0.03383466601371765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,8191,0.3877760171890259
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,8191,0.1943946679433187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,16383,0.37244800726572674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,16383,0.7544639905293783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,1,0.04015466570854187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,1,0.033674667278925575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,3,0.033759998778502144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,3,0.041109333435694374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,7,0.033861334125200905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,7,0.04146666576464971
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,15,0.04146133363246918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,15,0.03381866713364919
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,31,0.04176533222198486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,1,0.08161066472530365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,31,0.03389866650104523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,1,0.054144000013669334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,63,0.04285866518815359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,3,0.08255466818809509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,63,0.03416533271471659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,3,0.054117331902186074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,7,0.08322133123874664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,7,0.054272000988324486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,31,0.08523199955622356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,15,0.08343467116355896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,15,0.054234668612480164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,31,0.054560000697771706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,63,0.10847466190656026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,63,0.05537599821885427
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,1,0.13362666964530945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,1,0.09537600477536519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,3,0.13643733660380045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,3,0.0953653355439504
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,7,0.13781332969665527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,7,0.09537600477536519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,15,0.13859732945760092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,15,0.09544000029563904
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,31,0.1539253294467926
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,31,0.09599467118581136
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,63,0.09713600079218547
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,1,0.2364906668663025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,63,0.2709546685218811
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,1,0.17432000239690146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,3,0.24114133914311728
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,3,0.17406932512919107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,7,0.24524267514546713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,7,0.17443199952443442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,15,0.17497599124908447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,31,0.309386670589447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,15,0.26419732968012494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,31,0.17773866653442383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,63,0.4779413143793742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,63,0.23549334208170572
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,1,0.07188266515731812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,1,0.05841066439946493
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,3,0.07314133147398631
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,3,0.05829333265622457
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,7,0.07408000032107036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,7,0.05839466551939646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,15,0.0740533322095871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,15,0.05829866727193197
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,31,0.08102933565775554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,31,0.05855466425418854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,1,0.15793599685033163
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,1,0.09853333234786987
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,3,0.16831467549006143
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,3,0.09868799646695454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,7,0.17857599258422852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,7,0.09851200381914775
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,15,0.20105600357055664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,15,0.09885332981745402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,1,0.26390933990478516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,31,0.09962667028109233
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,1,0.18392000595728555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,3,0.29063467184702557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,31,0.23382399479548135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,7,0.31785066922505695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,7,0.18416533867518106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,3,0.18372267484664917
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,15,0.3279413382212321
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,15,0.18475733200709024
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,31,0.37584535280863446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,31,0.19335466623306274
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,1,0.49117334683736164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,1,0.34172264734903973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,3,0.5472319920857748
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,3,0.3417760133743286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,1,0.01137599969903628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1,0.009152000149091085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,3,0.009045333291093508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,3,0.011594666788975397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,7,0.011434666812419891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,7,0.009152000149091085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,15,0.009279999881982803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,15,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,31,0.009237333511312803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,31,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,63,0.00933333362142245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,63,0.011359999577204386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,127,0.009557333464423815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,127,0.011567999919255575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,7,0.343173344930013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,7,0.5629066626230875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,255,0.013093333691358566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,255,0.012805332740147909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,511,0.013573333621025085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1023,0.015024000157912573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,2047,0.021546666820844013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,1023,0.01758933315674464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,511,0.01461333284775416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,2047,0.023402666052182514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,4095,0.026634665826956432
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,8191,0.0365280012289683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,8191,0.05906666815280914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,4095,0.035375999907652535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,15,0.5655626853307089
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,1,0.011600000162919363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,3,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1,0.012773333738247553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,3,0.011488000551859537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,15,0.012847999731699625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,15,0.34916265805562335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,15,0.01145600030819575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,31,0.011557333171367645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,31,0.012960000584522883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,63,0.01302933320403099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,63,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,7,0.011802667131026586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,127,0.013365333278973898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,255,0.015770666301250458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,255,0.01312000056107839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,511,0.014682666709025701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,511,0.016735999534527462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1023,0.02218666672706604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,1023,0.01766933376590411
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,2047,0.0271573339899381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,4095,0.041152000427246094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,4095,0.03579200059175491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,8191,0.19947733481725058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,8191,0.05949866771697998
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,2047,0.023914667467276256
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1,0.012869333227475485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,127,0.01192533348997434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1,0.011765333513418833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,3,0.012917333592971167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,3,0.01301866645614306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,7,0.011695999652147293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,7,0.013066666821638743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,15,0.013045333325862885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,63,0.013173333058754602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,31,0.011722666521867117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,15,0.011621333658695221
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,31,0.01313599944114685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,63,0.011765333513418833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,31,0.6084693272908529
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,127,0.013647999614477158
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,127,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,255,0.016127999871969223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,511,0.02139200021823247
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,255,0.013631999492645264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,511,0.015157333264748255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1023,0.026485333840052288
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,2047,0.035802667339642845
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1023,0.018138666947682697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,4095,0.20287466049194336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,4095,0.03706666578849157
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1,0.012960000584522883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1,0.013797332843144735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,8191,0.1930933396021525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,3,0.013141332815090815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,3,0.013786666095256805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,8191,0.3906559944152832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,2047,0.02475200096766154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,7,0.01321600005030632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,7,0.013530666629473368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,15,0.013690666606028875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,15,0.01320533330241839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,31,0.013349333157142004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,31,0.013807999591032663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,127,0.01403733342885971
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,127,0.014666666587193808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,63,0.013722666849692663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,63,0.013855999956528345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,255,0.021253332495689392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,511,0.025909334421157837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,255,0.016336000214020412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1023,0.03573866685231527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,511,0.01907733331123988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,31,0.36536534627278644
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1023,0.024192000428835552
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,2047,0.20212799310684204
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,2047,0.03444266567627589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,4095,0.38783466815948486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,4095,0.19546133279800415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,8191,0.7567093372344971
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1,0.009136000027259191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,1,0.01179733375708262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,8191,0.37327468395233154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,3,0.011765333513418833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,7,0.00919999989370505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,7,0.011855999628702799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,15,0.011866666376590729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,15,0.009568000212311745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,31,0.00938666673998038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,31,0.011722666521867117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,63,0.00960533320903778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,63,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,127,0.009685333197315535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,255,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,127,0.012074666718641916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,255,0.013669333110253016
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,511,0.016634666671355564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,511,0.01451733335852623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1023,0.020256000260512035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,1023,0.022485333184401195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,2047,0.034629332522551216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,2047,0.025216000775496166
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,8191,0.10604266325632732
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1,0.012869333227475485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,8191,0.19869865973790488
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,4095,0.03933866570393244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,4095,0.05842666824658712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1,0.011866666376590729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,3,0.009248000259200731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,7,0.012917333592971167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,3,0.011882666498422623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,7,0.011829332758982977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,3,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,31,0.013141332815090815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,15,0.013061333447694778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,31,0.01192533348997434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,63,0.013253333667914072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,15,0.01198400060335795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,63,0.011887999872366587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,127,0.013637332866589228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,127,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,255,0.01640533283352852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,255,0.013850666582584381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,511,0.02181866765022278
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,511,0.016805333395799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1023,0.026757332185904186
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1023,0.02298133323589961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,4095,0.19857599337895712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,4095,0.05871466795603434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,2047,0.03664000084002813
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,2047,0.03478399912516276
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,8191,0.3813333511352539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1,0.01202133297920227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,8191,0.19323732455571493
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,3,0.013173333058754602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,7,0.012181332955757776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,15,0.013088000317414602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1,0.012981332838535309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,3,0.012138667205969492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,31,0.01328533391157786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,7,0.013269333789745966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,15,0.01190399999419848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,31,0.012096000214417776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,63,0.013381333400805792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,127,0.014127999544143677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,63,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,127,0.012730666746695837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,255,0.02120000123977661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,255,0.014287999520699183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,511,0.026170666019121807
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,511,0.017610666652520496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,2047,0.20267200469970703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,2047,0.03606933355331421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1023,0.03617066641648611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1023,0.023669332265853882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,4095,0.19334399700164795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,4095,0.3909279902776082
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,8191,0.37009068330128986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1,0.016741332908471424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,8191,0.7564373016357422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1,0.014175999909639359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,3,0.014261333892742792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,7,0.01414399966597557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,15,0.017008000363906223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,31,0.017237332959969837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,15,0.014202666779359182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,7,0.017008000363906223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,3,0.01682666689157486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,31,0.014266667266686758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,63,0.017360000560681026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,63,0.01470400020480156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,127,0.01871466636657715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,255,0.023472001155217487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,255,0.018351999421914417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,127,0.015530666957298914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,511,0.03321066747109095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1023,0.19947733481725058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1023,0.03379199902216593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,2047,0.3869280020395915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,2047,0.19541333119074503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1,0.008799999952316284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,1,0.01073066641887029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,4095,0.753770669301351
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,4095,0.3730613390604655
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,511,0.02370133250951767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,3,0.008618666479984919
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,3,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,7,0.008746666833758354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,8191,0.7282293637593588
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,7,0.010762666662534079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,15,0.00877333308259646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,31,0.010741333166758219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,31,0.00884799969693025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,8191,1.488485336303711
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,63,0.009008000294367472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,63,0.010735999792814255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,127,0.00916800027092298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,15,0.01080000028014183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,127,0.010725333044926325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,255,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,255,0.011498666057984034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,511,0.01173866664369901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,2047,0.01440000037352244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1023,0.013376000026861826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,1023,0.011727999895811081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,511,0.012773333738247553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,4095,0.016656000167131424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,2047,0.011823999385039011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,8191,0.013733333597580591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,16383,0.031119999786218006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,4095,0.012128000458081564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,16383,0.01670933390657107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,8191,0.02128533273935318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,32767,0.022831998765468597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1,0.00867733359336853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,32767,0.033370666205883026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,3,0.00860799973209699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1,0.010698666175206503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,3,0.010869332899649939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,7,0.008789333204428354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,15,0.008714666590094566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,7,0.010608000059922537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,31,0.00879466657837232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,63,0.009125333279371262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,31,0.010858666151762009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,63,0.010869332899649939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,127,0.009082666908701261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,15,0.010746666540702185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,255,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,127,0.010901333143313726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,511,0.012421333541472753
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,255,0.011706666400035223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,511,0.01181866725285848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1023,0.013189333180586496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1023,0.011674666156371435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,2047,0.014303999642531076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,2047,0.011952000359694162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,4095,0.016741332908471424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,4095,0.012602667013804117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,8191,0.02160533269246419
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,8191,0.013909333695967993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,16383,0.035114665826161705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,16383,0.016917333006858826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,32767,0.023344000180562336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1,0.00871999996403853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,3,0.008672000219424566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1,0.010709332923094431
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,32767,0.0402453343073527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,7,0.008789333204428354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,3,0.01080000028014183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,15,0.009039999917149544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,7,0.010773333410422007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,31,0.008896000062425932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,15,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,63,0.00890666681031386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,127,0.009248000259200731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,63,0.010666667173306147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,255,0.012426666915416718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,127,0.01090666651725769
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,31,0.010645333677530289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,511,0.012655999511480331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,255,0.011578666667143503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1023,0.013232000172138214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,511,0.011930666863918304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,2047,0.014442666123310724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1023,0.012005332857370377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,4095,0.01732800031701724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,2047,0.012266666938861212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,8191,0.026250667870044708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,4095,0.012928000340859095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,16383,0.03133333226044973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,8191,0.014453332871198654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,32767,0.04087999959786733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,16383,0.017610666652520496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1,0.008645333349704742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,3,0.008789333204428354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,32767,0.023925334215164185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,3,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1,0.011600000162919363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,15,0.008858666444818178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,7,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,7,0.008896000062425932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,15,0.012053333222866058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,63,0.00902399979531765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,31,0.00879466657837232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,63,0.01109333336353302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,255,0.01219733307758967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,31,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,255,0.011994666109482447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,127,0.00916800027092298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,127,0.01145600030819575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1023,0.013429333766301474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,511,0.01379199946920077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,511,0.012741333494583765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,2047,0.015066667149464289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,8191,0.026874666412671406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1023,0.014053333550691605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,4095,0.015770666301250458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,4095,0.02165866643190384
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,16383,0.03664533297220866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,2047,0.014655999839305878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,8191,0.01834133391578992
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,16383,0.02348266790310542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,32767,0.034186666210492454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,3,0.010821333775917688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1,0.008592000231146812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,3,0.008885333314538002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,32767,0.22177066405614218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,7,0.008757333581646284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,7,0.010725333044926325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,15,0.008752000207702318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,15,0.010901333143313726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,31,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,31,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,63,0.00901333304742972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,63,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,255,0.01258133351802826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,127,0.009082666908701261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,127,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,255,0.011653333902359009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,511,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,2047,0.014720000326633453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,511,0.012768000364303589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,1023,0.011749333391586939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1023,0.013429333766301474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,2047,0.011994666109482447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,4095,0.013674666484196981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,8191,0.016528000434239704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,8191,0.02197333425283432
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,4095,0.01685333376129468
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,16383,0.03584533433119456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,16383,0.022554665803909302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,32767,0.04139200101296107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,32767,0.03461333364248276
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1,0.012096000214417776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,3,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,3,0.010746666540702185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,7,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,15,0.012080000092585882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,7,0.010709332923094431
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,31,0.0122079998254776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,15,0.010869332899649939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,31,0.010928000013033548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,63,0.012319999436537424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,127,0.01249066616098086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,63,0.010832000523805618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,255,0.015103999525308609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,127,0.011050666371981302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,255,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,511,0.015173333386580149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,511,0.011658667276302973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1023,0.015754666179418564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,2047,0.017152000218629837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1023,0.012015999605258306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,2047,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,4095,0.019893333315849304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,8191,0.028901333610216778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,4095,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,16383,0.03425066669782003
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,8191,0.016719999412695568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,32767,0.044031997521718345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,16383,0.023013333479563396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1,0.011989332735538483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,3,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,32767,0.03491200009981791
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,7,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,3,0.010858666151762009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,15,0.012191999703645706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,7,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,31,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,63,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,127,0.012533333152532578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,63,0.010821333775917688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,255,0.01482133318980535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,127,0.011237333218256632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,511,0.015301333119471868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,511,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1023,0.01613866661985715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,2047,0.01764800027012825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1023,0.01202133297920227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,2047,0.012906666845083237
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,255,0.012053333222866058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,4095,0.024506665766239166
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,8191,0.029535998900731403
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,4095,0.014362666755914688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,16383,0.03925866633653641
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,8191,0.017290666699409485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,16383,0.023706667125225067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,32767,0.036415999134381614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,3,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,32767,0.2250880002975464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,3,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,15,0.012165332833925882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,7,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,7,0.011498666057984034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,15,0.011194666226704916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,31,0.011813333878914515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,63,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,63,0.01146666705608368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,31,0.012400000045696894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,127,0.012357333054145178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,127,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,255,0.015050667027632395
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,255,0.014058666924635569
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,511,0.014202666779359182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,2047,0.0220266655087471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,511,0.015360000232855478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1023,0.01659199967980385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1023,0.014544000228246054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,4095,0.02722666660944621
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,2047,0.015583999454975128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,4095,0.018186666071414948
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,16383,0.2022613286972046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,8191,0.02334933231274287
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,8191,0.03693866729736328
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,16383,0.03342933456103007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,32767,0.38787734508514404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,32767,0.19427732626597086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1,0.009514666472872099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,1,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,3,0.009472000102202097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,3,0.012400000045696894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,7,0.009568000212311745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,7,0.012543999900420507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,15,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,15,0.009642666826645533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,31,0.009568000212311745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,31,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,63,0.009898666913310686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,63,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,127,0.013157332936922709
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,255,0.015013333410024643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,255,0.0143306665122509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,127,0.010506667196750641
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,511,0.020554666717847187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,511,0.017583999782800674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1023,0.025407999753952026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,1023,0.023344000180562336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,2047,0.035391998787721
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1,0.013530666629473368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,2047,0.03540800015131632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1,0.012645332763592402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,3,0.013669333110253016
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,4095,0.05931200087070465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,7,0.013717333475748697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,4095,0.20014933745066324
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,3,0.012671999633312225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,15,0.0136266661187013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,7,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,15,0.012762666990359625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,31,0.012629333883523941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,63,0.013034666577974955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,127,0.013295999417702356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,127,0.014554666976133982
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,31,0.013658666362365087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,255,0.022330666581789654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,255,0.014981333166360855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,511,0.027456000447273254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,63,0.014208000153303146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,511,0.01800000046690305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,4095,0.38384532928466797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1023,0.0372533326347669
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1023,0.024277334411938984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1,0.017221332838137943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,4095,0.19498666127522787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1,0.014522666732470194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,2047,0.20094933112462363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,3,0.017338667064905167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,2047,0.03712533414363861
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,7,0.017375999440749485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,3,0.014650666465361914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,7,0.014869333555301031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,15,0.017509333789348602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,31,0.014666666587193808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,15,0.014618666221698126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,63,0.01524266724785169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,31,0.017632000148296356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,127,0.019018666197856266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,127,0.01621333385507266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,255,0.024127999941507976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,255,0.018800000349680584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,511,0.03387733300526937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,63,0.01810666670401891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,2047,0.39049601554870605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,2047,0.19803732633590698
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,511,0.023925334215164185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,4095,0.7594239711761475
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1023,0.03409600009520849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,3,0.024538666009902954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1,0.020799999435742695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1,0.024271999796231587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1023,0.20201599597930908
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,4095,0.37546666463216144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,3,0.020714666694402695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,7,0.024906667570273083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,15,0.024800000091393787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,15,0.02090666691462199
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,31,0.02518933266401291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,63,0.025279998779296875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,63,0.021173333128293354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,7,0.02060266708334287
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,127,0.022618666291236877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,127,0.027189334233601887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,31,0.02092266579469045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,255,0.02787200113137563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1023,0.4275999863942464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,255,0.03724266588687897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,511,0.2025173306465149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,511,0.037248000502586365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1023,0.19583467642466226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,2047,0.8422666390736898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,3,0.008922666932145754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,3,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,7,0.008826666822036108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,2047,0.3763999938964844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,7,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,15,0.008901333436369896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,15,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,4095,1.607418696085612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1,0.008714666590094566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,1,0.011071999867757162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,4095,0.7323466936747233
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,63,0.009312000125646591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,31,0.011205332974592844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,31,0.008821333448092142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,127,0.009178666397929192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,63,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,127,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,511,0.01268799975514412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,511,0.012080000092585882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,255,0.011941333611806234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1023,0.013424000392357508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,2047,0.014949332922697067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,255,0.012613333761692047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,1023,0.012069333344697952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,2047,0.013701333353916803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,8191,0.0273333340883255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,4095,0.01756799966096878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,8191,0.02254933367172877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,16383,0.03214933226505915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,4095,0.016757333030303318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,16383,0.034501334031422935
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,32767,0.04218666752179464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,32767,0.05813866853713989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1,0.011274666835864386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1,0.012154666086037954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,3,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,3,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,7,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,7,0.012175999581813812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,15,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,31,0.012256000190973282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,15,0.01109333336353302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,31,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,63,0.011205332974592844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,127,0.012602667013804117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,63,0.012479999413092932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,255,0.014874666929244995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,127,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,255,0.011952000359694162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,511,0.01534933348496755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,511,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1023,0.012309333930412928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,4095,0.024570666253566742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,2047,0.017610666652520496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,2047,0.013744000345468521
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,4095,0.017045332739750545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1023,0.0161920003592968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,8191,0.029690665503342945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,8191,0.022783999641736347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,16383,0.0394400010506312
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,16383,0.03503466645876566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,32767,0.058917333682378135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,32767,0.19974400599797568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,3,0.012272000312805176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,3,0.011114666859308878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,7,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,15,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,31,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,15,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,63,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,31,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,63,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,127,0.012554666648308435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,127,0.011594666788975397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,255,0.014949332922697067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,255,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,511,0.015306666493415833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,511,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1023,0.01666133354107539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,2047,0.022197333474953968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1023,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,4095,0.027349332968393963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,2047,0.014368000129858652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,4095,0.017418666432301205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,8191,0.03722133239110311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,8191,0.023733332753181458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,16383,0.20268799861272177
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,16383,0.03571200122435888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1,0.012026666353146235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,32767,0.19220266739527384
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,32767,0.38891732692718506
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1,0.013167999684810638
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,7,0.012293333808581034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,3,0.01313599944114685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,3,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,7,0.01313599944114685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,15,0.013253333667914072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,31,0.012304000556468964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,31,0.013173333058754602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,15,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,63,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,63,0.01314666618903478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,127,0.013552000125249227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,127,0.012671999633312225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,511,0.01590399940808614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,255,0.015477333217859268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,255,0.014469332993030548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,511,0.014730667074521383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1023,0.015520000209410986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1023,0.021317332983016968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,2047,0.02629866699377696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,4095,0.02345066765944163
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,2047,0.018320000420014065
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,8191,0.03359466542800268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,4095,0.04041066765785217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,8191,0.22048000494639078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,16383,0.3880319992701213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,16383,0.20732800165812174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,32767,0.37147200107574463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,32767,0.7563626766204834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1,0.009637333452701569
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,1,0.013690666606028875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,3,0.009930666536092758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,3,0.013503999759753546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,7,0.010010666524370512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,7,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,15,0.00985599992175897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,15,0.013381333400805792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,31,0.010079999764760336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,31,0.013541333377361298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,63,0.01027199998497963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,63,0.01381333296497663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,127,0.010928000013033548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,127,0.01413333291808764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,255,0.021269333859284718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,255,0.015919999529918034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,511,0.02640533447265625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,511,0.018954666952292126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1023,0.03619199991226196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,1023,0.02526933451493581
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1,0.018079999834299088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,2047,0.2029013236363729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1,0.01569066693385442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,3,0.0182239996890227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,2047,0.037477334340413414
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,3,0.015610666324694952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,7,0.018426666657129925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,7,0.015669333438078564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,15,0.01842133328318596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,31,0.018559999763965607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,31,0.01570133368174235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,63,0.01607999950647354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,15,0.015557333827018738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,127,0.020031999796628952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,127,0.01727466657757759
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,63,0.01878400022784869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,255,0.024986666937669117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,255,0.019600000232458115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,511,0.03504000107447306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,511,0.02495466669400533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1,0.025226667523384094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1,0.02160533269246419
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1023,0.19716266791025797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,3,0.025557334224383037
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1023,0.03531199942032496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,7,0.025797332326571148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,7,0.021520001192887623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,3,0.021509334444999695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,2047,0.2009119987487793
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,2047,0.3805280129114787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,15,0.02165866643190384
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,15,0.025839999318122864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,31,0.02183466653029124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,31,0.02601066728432973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,63,0.026335999369621277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,63,0.02186133215824763
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,255,0.038202665746212006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,255,0.028522667785485584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,511,0.20499199628829956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,127,0.023344000180562336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,127,0.02811199923356374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,511,0.03867200016975403
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,2047,0.7667733033498129
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,2047,0.381279985109965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1,0.039066667358080544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1,0.03305600086847941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,7,0.04009599983692169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1023,0.2116533319155375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1023,0.39632534980773926
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,7,0.03311466674009959
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,3,0.03956799954175949
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,3,0.03331733246644338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,15,0.040149333576361336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,15,0.03306133300065994
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,31,0.04089066634575526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,63,0.03347733368476232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,63,0.041402667760849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,127,0.04353066782156626
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,127,0.03442133218050003
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,31,0.03327466547489166
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,255,0.045125335454940796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,255,0.22401599089304605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1023,0.7692000071207682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1023,0.38151466846466064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,511,0.19985600312550864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,511,0.39760533968607586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,2047,0.7376960118611654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,2047,1.5827147165934246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,1,0.01394133393963178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,1,0.017221332838137943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,3,0.014245333770910898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,7,0.017312000195185345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,15,0.014362666755914688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,15,0.017312000195185345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,31,0.01461333284775416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,31,0.017269333203633625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,3,0.017152000218629837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,7,0.014282666146755219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,63,0.014815999815861383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,63,0.017786666750907898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,127,0.018485333770513535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,127,0.016208000481128693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,255,0.02128533273935318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,255,0.021210665504137676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,1,0.027034667630990345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,511,0.03107733279466629
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,511,0.02657066782315572
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,1,0.023344000180562336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,3,0.027376001079877216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,3,0.023413332800070446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,7,0.027850667635599773
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,7,0.0233599990606308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,15,0.02770666778087616
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,15,0.023333333432674408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,31,0.023567999402681988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,31,0.0281333327293396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,63,0.028399998943010967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,63,0.023845332364241283
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,127,0.030421334008375805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,127,0.025562666356563568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,255,0.04009066770474116
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,255,0.03054400036732356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,1,0.041178666055202484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,1,0.034832000732421875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,511,0.20468799273173013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,511,0.040661332507928215
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,3,0.04200000067551931
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,3,0.03489600121974945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,7,0.04228800038496653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,15,0.04228800038496653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,7,0.034714666505654655
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,15,0.034901333351929985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,31,0.0428959975639979
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,31,0.034917332231998444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,63,0.04372799893220266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,63,0.03523733218510946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,127,0.04557866851488749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,127,0.03595733394225439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,1,0.06589866677920024
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,255,0.04659200211366018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,255,0.21224000056584677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,1,0.05455466608206431
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,511,0.40662399927775067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,511,0.20637333393096924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,3,0.054341331124305725
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,3,0.0668213317791621
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,7,0.06791999936103821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,15,0.054661333560943604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,15,0.06803733110427856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,7,0.054325332244237266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,63,0.06985599795977275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,63,0.05490666627883911
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,127,0.21218132972717285
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,31,0.05444799860318502
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,31,0.06890666484832764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,127,0.05731200178464254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,255,0.40666667620340985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,255,0.20660267273585
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,1,0.027066667874654133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,3,0.022895999252796173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,511,0.780944029490153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,511,0.3891893227895101
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,1,0.0229066660006841
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,3,0.02717866748571396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,7,0.027210667729377747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,15,0.023061332603295643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,15,0.02720533311367035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,7,0.022997332115968067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,63,0.023887999355793
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,63,0.027786667148272198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,31,0.027077332139015198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,31,0.023370665808518726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,127,0.026026666164398193
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,127,0.029338667790095013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,255,0.036042665441830955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,1,0.04489600161711375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,255,0.034661332766215004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,1,0.03869866579771042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,3,0.04549333453178406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,3,0.03914133210976919
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,7,0.04598399996757507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,7,0.03895466774702072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,15,0.045978665351867676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,15,0.03902933249870936
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,31,0.046575998266537987
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,31,0.03899733225504557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,63,0.04762133459250132
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,63,0.0395413339138031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,127,0.049509331583976746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,127,0.04037333279848099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,255,0.2172266642252604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,255,0.0516533354918162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,1,0.07061866422494252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,1,0.05956799785296122
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,3,0.05963733295599619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,3,0.07180266578992207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,7,0.0728053351243337
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,7,0.05979733169078827
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,15,0.07296533385912578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,31,0.07335466643174489
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,15,0.06007466713587443
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,31,0.06011199951171875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,63,0.07477333148320515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,63,0.06063466767470042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,127,0.22499199708302817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,127,0.06342400113741557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,1,0.12089066704114278
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,1,0.09910933176676433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,255,0.22054400046666464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,255,0.4251360098520915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,3,0.09966933727264404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,3,0.12301333745320638
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,7,0.12478933731714885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,15,0.12473600109418233
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,7,0.09939733147621155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,15,0.09956799944241841
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,31,0.12650133172671
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,63,0.24594666560490927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,63,0.10063466429710388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,31,0.09993066390355428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1,0.00892800030608972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,7,0.00895999992887179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,7,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,3,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,1,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,15,0.00891733355820179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,15,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,3,0.009237333511312803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,31,0.00903466654320558
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,31,0.01118933285276095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,63,0.011317333827416102
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,63,0.009296000003814697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,127,0.009418666362762451
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,127,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,255,0.012725333372751871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,511,0.012965332716703415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,511,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,255,0.012293333808581034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1023,0.01403733342885971
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,1023,0.013733333597580591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,2047,0.01571200042963028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,2047,0.016901332885026932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,4095,0.022783999641736347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,127,0.42512532075246173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,4095,0.02276800076166789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,8191,0.027808000644048054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,8191,0.03477333237727483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,16383,0.03736533224582672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,32767,0.10609599947929382
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1,0.012485332787036896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,32767,0.20060267051060995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,127,0.22249066829681396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,3,0.01249066616098086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,7,0.012586666891972223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,3,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,7,0.011274666835864386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,15,0.011274666835864386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,31,0.012624000509579977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,31,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,63,0.012874666601419449
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,63,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,127,0.012965332716703415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,255,0.015189333508412043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,127,0.011626667032639185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,255,0.012191999703645706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,511,0.015728000551462173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,511,0.012448000411192576
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1023,0.016917333006858826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1023,0.013904000322024027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,2047,0.022687998910744984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,2047,0.017071999609470367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,16383,0.058602665861447654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,4095,0.027642667293548584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,4095,0.023061332603295643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,8191,0.035002666215101876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,8191,0.037802666425704956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,16383,0.19870932896931967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,16383,0.05890133480230967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,255,0.43111467361450195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,15,0.012741333494583765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,255,0.8036159674326578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1,0.012469333906968435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,32767,0.19299199183781943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,32767,0.3813333511352539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,3,0.012671999633312225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,7,0.012624000509579977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,31,0.012666666259368261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,7,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,15,0.011296000331640244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,31,0.011482667177915573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,63,0.012719999998807907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,63,0.011429333438475927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,127,0.011695999652147293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,255,0.015562667200962702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,255,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,127,0.013088000317414602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,511,0.016309333344300587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,511,0.012885333349307379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1023,0.021551998953024547
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1023,0.01440000037352244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,2047,0.026608000199000042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,2047,0.017925333231687546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,4095,0.0405973345041275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,4095,0.023957334458827972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,8191,0.03636800001064936
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,8191,0.20246400435765585
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,16383,0.3909706672032674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,16383,0.1923733353614807
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,32767,0.3684213161468506
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1,0.013349333157142004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,32767,0.7584213415781657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,3,0.012762666990359625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,3,0.013248000293970108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,7,0.013530666629473368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,15,0.012874666601419449
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,7,0.012826666235923767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,31,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,15,0.013461332768201828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,63,0.013093333691358566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,31,0.013455999394257864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,127,0.013301332791646322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,63,0.013610667238632837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,3,0.011391999820868174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,127,0.01392000044385592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,255,0.016058667252461117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,255,0.01498666654030482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1023,0.02588266630967458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,511,0.020741333564122517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,2047,0.03577066709597906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,511,0.01575999955336253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1023,0.01851733277241389
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,15,0.012768000364303589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,2047,0.023765332996845245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,4095,0.20094933112462363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,4095,0.03346133232116699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,8191,0.38681598504384357
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,8191,0.1946400006612142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,16383,0.755354642868042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,16383,0.3997653325398763
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,32767,0.7256266276041666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,32767,1.6672426859537761
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,1,0.03870933254559835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,1,0.03297599901755651
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,3,0.0396373321612676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,7,0.039919999738534294
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,3,0.03287466615438461
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,7,0.03307733436425527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,15,0.040175999204317726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,15,0.03299200038115183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,31,0.040565334260463715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,1,0.07783466577529907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,31,0.03321066747109095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,1,0.05323199927806854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,63,0.04127466678619385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,63,0.03356266766786575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,127,0.03444266567627589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,127,0.04459733267625173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,3,0.0787306676308314
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,3,0.05320533116658529
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,7,0.07975466549396515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,7,0.05343466500441233
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,15,0.07971733311812083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,31,0.08094933132330577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,15,0.05349333087603251
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,31,0.05366933345794678
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,63,0.08960533142089844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,1,0.1302720010280609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,63,0.05436799923578898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,1,0.09433600306510925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,127,0.05779199798901876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,127,0.2462559938430786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,3,0.13218133648236594
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,3,0.09424533446629842
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,7,0.1341493328412374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,7,0.09442133704821269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,15,0.13412800431251526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,15,0.0946613351504008
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,31,0.09512000282605489
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,31,0.14332800110181174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,63,0.2550666729609172
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,63,0.09590400258700053
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,1,0.23226666450500488
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,1,0.17338132858276367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,127,0.22619199752807617
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,127,0.46058666706085205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,3,0.1732213298479716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,3,0.23652799924214682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,7,0.24014399449030557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,7,0.17318934202194214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,15,0.17376534144083658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,15,0.25084267059961957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,31,0.28684266408284503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,31,0.17564266920089722
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,63,0.46112533410390216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,63,0.2269973357518514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,127,0.8613653182983398
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,127,0.45238931973775226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,1,0.06897599995136261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,1,0.05667200187842051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,3,0.07018666466077168
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,3,0.056741332014401756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,7,0.056703999638557434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,7,0.07100800176461537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,15,0.07109866539637248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,15,0.056661332647005715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,31,0.07341333230336507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,31,0.05718400080998739
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,63,0.13274666666984558
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,1,0.1451573371887207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,63,0.05761066575845083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,1,0.09693866968154907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,3,0.1481119990348816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,3,0.0967733363310496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,7,0.15421332915623984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,7,0.09711999694506328
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,15,0.1684266726175944
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,15,0.0974133312702179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,31,0.2020960052808126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,31,0.09775466720263164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,63,0.3115573326746623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,63,0.105813334385554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,1,0.2510026693344116
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,1,0.18200532595316568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,3,0.2581440011660258
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,3,0.1819253365198771
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,7,0.28222399950027466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,7,0.18205867211023966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,15,0.3040906588236491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,15,0.18225600322087607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,31,0.3447893460591634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,31,0.18756266434987387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,63,0.5331253210703532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,1,0.458842674891154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,63,0.26450133323669434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,1,0.33986131350199383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,3,0.4968693256378174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,3,0.339408000310262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,7,0.5281866788864136
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,7,0.3402080138524373
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1,0.009018666421373686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,1,0.011418666690587997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,3,0.011381333072980246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,7,0.009136000027259191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,7,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,3,0.009114666531483332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,15,0.009119999905427298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,31,0.009194666519761086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,15,0.011296000331640244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,15,0.5338666836420695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,31,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,63,0.009301333377758661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,63,0.01137599969903628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,127,0.009493333597977957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,255,0.013013333082199097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,255,0.012634667257467905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,31,0.5814719994862875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,15,0.3450666666030884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,127,0.011600000162919363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,511,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,511,0.014015999933083853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,1023,0.01720000058412552
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1023,0.014639999717473984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,2047,0.02314666658639908
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,4095,0.03513599932193756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,8191,0.05893866717815399
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,8191,0.03588266670703888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,31,0.36289600531260174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,16383,0.19965867201487222
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,16383,0.10659733414649963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1,0.011589333415031433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,4095,0.026335999369621277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,3,0.012746666868527731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,2047,0.021168000996112823
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,3,0.011642667154471079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,7,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,7,0.012997332960367203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,15,0.013013333082199097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,15,0.011525332927703857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,31,0.012826666235923767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,31,0.011514666179815928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,63,0.013130666067202887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1,0.012949333836634954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,63,0.011690666278203329
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,127,0.013253333667914072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,127,0.011893333246310553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,255,0.015893333901961643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,255,0.013077333569526672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,511,0.014357333381970724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1023,0.022111999491850536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,511,0.01664000004529953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1023,0.0174346665541331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,2047,0.023503998915354412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,4095,0.04114133367935816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,2047,0.027136000494162243
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,4095,0.03557866563399633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,8191,0.1978879968325297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,63,0.9242186546325684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1,0.012863999853531519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,8191,0.05929600199063619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,16383,0.382421334584554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,16383,0.19419199228286743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1,0.011674666156371435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,3,0.012831999609867731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,7,0.013034666577974955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,3,0.011551999797423681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,15,0.011546666423479715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,31,0.012874666601419449
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,15,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,31,0.011616000284751257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,63,0.011674666156371435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,63,0.013104000439246496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,127,0.01357866699496905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,255,0.016330666840076447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,127,0.011957333733638128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,255,0.013424000392357508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,511,0.014970666418472925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,511,0.021146667500336964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,2047,0.03606399893760681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1023,0.026367999613285065
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1023,0.01817600056529045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,2047,0.024288001159826916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,4095,0.036661334335803986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,4095,0.20245865980784097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,63,0.5062613487243652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,8191,0.1930239995320638
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,8191,0.3893760045369466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1,0.013088000317414602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,16383,0.36939199765523273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,16383,0.8373066584269205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1,0.01357866699496905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,3,0.013077333569526672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,7,0.013157332936922709
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,3,0.01350933313369751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,15,0.01358933374285698
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,63,0.013471999516089758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,31,0.01332266628742218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,31,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,63,0.013946666071812311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,7,0.011765333513418833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,127,0.014080000420411428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,127,0.0144213338692983
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,255,0.021045332153638203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1023,0.035775999228159584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1023,0.0239680012067159
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,511,0.026234666506449383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,255,0.01637866720557213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,511,0.018778666853904724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,2047,0.034389334420363106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,2047,0.22037333250045776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,4095,0.3876906633377075
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,4095,0.1950826644897461
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,8191,0.37299199899037677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,1,0.011616000284751257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1,0.008997333546479544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,8191,0.7545546690622965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,3,0.009242666885256767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,16383,0.7277173201243082
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,3,0.011557333171367645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,16383,1.4898239771525066
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,7,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,7,0.01357866699496905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,7,0.009232000137368837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,15,0.011551999797423681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,15,0.009312000125646591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,31,0.00927466650803884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,31,0.011663999408483505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,63,0.011642667154471079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,63,0.009317333499590555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,127,0.011936000237862269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,255,0.013434667140245438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,127,0.009583999713261923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,255,0.013274667163689932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,511,0.014175999909639359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,511,0.016384000579516094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,1023,0.02252800017595291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,2047,0.02476799984773
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1023,0.01977066695690155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,2047,0.03417599946260452
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,8191,0.1058240036169688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,8191,0.21675733725229898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,4095,0.038586666186650596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,4095,0.058304001887639366
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1,0.011621333658695221
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,3,0.011637333780527115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,3,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,7,0.01303999995191892
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,15,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,31,0.013023999830087027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,15,0.013045333325862885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,7,0.011839999506870905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,31,0.011711999773979187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,63,0.013130666067202887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,63,0.011770666887362799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,127,0.013514666507641474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,127,0.011999999483426413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,255,0.016208000481128693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,255,0.013829333086808523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,511,0.021312000850836437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,511,0.016554666062196095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1023,0.026399999856948853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1023,0.022704000274340313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,4095,0.19801066319147745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,2047,0.036271999279658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,4095,0.058575997749964394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,2047,0.03469866762558619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,8191,0.37985066572825116
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1,0.012879999975363413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,8191,0.19173866510391235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1,0.012063999970753988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,3,0.011861333002646765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,3,0.013077333569526672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,7,0.013183999806642532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,15,0.01312000056107839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,7,0.011781333635250727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,31,0.013343999783198038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,31,0.011920000116030375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,15,0.011882666498422623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,63,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,127,0.013914667069911957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,63,0.012037333101034164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,127,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,255,0.021087999145189922
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,255,0.014250667144854864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,511,0.026181332767009735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,511,0.017317333569129307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,2047,0.03594133257865906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1023,0.03579200059175491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1023,0.023434666295846302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,2047,0.2210293412208557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,4095,0.3885226647059123
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,4095,0.19268266359965006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,8191,0.3676160176595052
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,3,0.013888000200192133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,3,0.01672533278663953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,8191,0.7555147012074789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,7,0.017055999487638474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1,0.016575999557971954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1,0.013888000200192133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,7,0.014021333307027817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,15,0.016879999389251072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,15,0.013909333695967993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,31,0.01714133347074191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,31,0.014021333307027817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,63,0.01443733274936676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,63,0.017450666675964992
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,127,0.0184906671444575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,127,0.015493333339691162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,255,0.018101333330074947
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1023,0.19900800784428915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1023,0.033573334415753685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,255,0.02348266790310542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,511,0.03297066688537598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,511,0.02345066765944163
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,2047,0.194922665754954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,2047,0.42797334988911945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,4095,0.37169599533081055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1,0.00847999999920527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,3,0.008634666601816813
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,3,0.01073066641887029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,7,0.008602666358153025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,7,0.010640000303586325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,4095,0.7531200249989828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,15,0.008672000219424566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,8191,0.7258559862772623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,1,0.010661333799362183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,31,0.010602666685978571
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,8191,1.6675519943237305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,31,0.00874133345981439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,63,0.00874133345981439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,63,0.01071999967098236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,127,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,255,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,127,0.00895999992887179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,511,0.012549333274364471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,255,0.011706666400035223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,511,0.011525332927703857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,1023,0.01157333329319954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1023,0.01322666679819425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,2047,0.014314666390419006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,8191,0.013546666751305262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,8191,0.020997333029905956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,4095,0.016565332810084026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,4095,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,2047,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,16383,0.01658133293191592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,16383,0.03033066789309184
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,32767,0.03305600086847941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,32767,0.02279466638962428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,3,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,7,0.008698666468262672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1,0.00855466661353906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,3,0.008693333094318708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1,0.010816000401973724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,7,0.010645333677530289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,15,0.008687999720374743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,31,0.008826666822036108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,15,0.010602666685978571
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,31,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,63,0.008933333059151968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,63,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,127,0.00901333304742972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,255,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,127,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,511,0.012495999534924826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,255,0.011578666667143503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1023,0.013066666821638743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1023,0.011594666788975397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,2047,0.0144213338692983
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,511,0.011706666400035223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,2047,0.011952000359694162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,4095,0.016688000410795212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,4095,0.012304000556468964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,8191,0.021589333812395733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,8191,0.013834666460752487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,16383,0.03490666548411051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,32767,0.04019733270009359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1,0.008645333349704742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,32767,0.023290666441122692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,16383,0.0170666662355264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,3,0.008613333106040955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,7,0.008949333180983862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,15,0.009088000282645226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,3,0.010911999891201654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,7,0.010709332923094431
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,31,0.008879999940594038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,15,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,31,0.010634666929642359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,63,0.008810666700204214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,127,0.009183999771873156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,63,0.010693332801262537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,255,0.01246400053302447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,127,0.010954666882753372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,511,0.012698666503032049
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1023,0.013455999394257864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,511,0.011711999773979187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,2047,0.014688000082969666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,2047,0.012223999947309494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1023,0.011994666109482447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,4095,0.01717866708834966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,8191,0.026288000245889027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,4095,0.012757333616415659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,16383,0.03134933362404505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,32767,0.04093866546948751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,16383,0.017514667163292568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,8191,0.014618666221698126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,32767,0.024090667565663654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1,0.008629333227872849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,7,0.008832000195980072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,3,0.008767999708652496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,3,0.01102399950226148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,15,0.011071999867757162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,31,0.00895999992887179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,15,0.008943999807039896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,7,0.0116799995303154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,31,0.01118933285276095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,63,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,63,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,127,0.009178666397929192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,127,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,511,0.012741333494583765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1023,0.013418667018413544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,255,0.012549333274364471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1023,0.013994666437307993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,2047,0.015082667271296183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,511,0.013978666315476099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,255,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,2047,0.0145066666106383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,4095,0.021781332790851593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,4095,0.015573333948850632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,8191,0.026719999810059864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,8191,0.018645333747069042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,16383,0.023743999501069386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,16383,0.03658666710058848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,32767,0.20041600863138834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,32767,0.033887999753157295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1,0.008645333349704742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,1,0.0107893335322539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,3,0.008736000085870424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,7,0.010757333288590113
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,3,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,7,0.008698666468262672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,15,0.008832000195980072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,31,0.008949333180983862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,31,0.010842667271693548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,63,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,63,0.008933333059151968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,255,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,127,0.009061333412925402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,127,0.01118933285276095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,255,0.011658667276302973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,511,0.012826666235923767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,511,0.011813333878914515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,2047,0.014357333381970724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1023,0.013280000537633896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,1023,0.011770666887362799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,4095,0.016949333250522614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,2047,0.011952000359694162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,8191,0.02203733225663503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,8191,0.01655999943614006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,16383,0.035375999907652535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,4095,0.013610667238632837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,16383,0.02276800076166789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,32767,0.04063999901215235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,32767,0.034527999659379326
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1,0.012256000190973282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1,0.010928000013033548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,7,0.012181332955757776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,7,0.010762666662534079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,15,0.012202666451533636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,31,0.012335999558369318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,3,0.012058666596810022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,63,0.012245333443085352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,31,0.010762666662534079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,63,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,127,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,127,0.01110400011142095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,255,0.01482133318980535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,511,0.01173866664369901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,511,0.015103999525308609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,255,0.011936000237862269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1023,0.015856000284353893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1023,0.01190399999419848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,2047,0.01728533332546552
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,4095,0.019845332950353622
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,4095,0.013637332866589228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,2047,0.01209066684047381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,8191,0.028858666618665058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,16383,0.0341333324710528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,8191,0.016794666647911072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,16383,0.02293866624434789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,32767,0.043968002001444496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,32767,0.03489066660404205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1,0.01099733387430509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,3,0.010879999647537867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,7,0.012421333541472753
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1,0.012165332833925882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,15,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,7,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,3,0.012229333321253458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,15,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,31,0.012357333054145178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,127,0.012543999900420507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,63,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,63,0.01089599976936976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,255,0.014890667051076889
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,127,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,255,0.011973333855470022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,511,0.015295999745527903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,511,0.011999999483426413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1023,0.016000000139077503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,2047,0.017770666629076004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1023,0.012154666086037954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,2047,0.012768000364303589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,4095,0.02447466552257538
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,8191,0.029472000896930695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,4095,0.01423466702302297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,16383,0.03941866755485535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,8191,0.017269333203633625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,16383,0.02348266790310542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,32767,0.2053333322207133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,32767,0.03642133375008901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,3,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,3,0.011541333049535751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,7,0.011381333072980246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,15,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,7,0.012341332932313284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,15,0.011866666376590729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,31,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,63,0.012426666915416718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,127,0.012586666891972223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,63,0.01157333329319954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,31,0.011482667177915573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,127,0.011658667276302973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,255,0.013957332819700241
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,255,0.014885333677132925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,511,0.015477333217859268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,2047,0.022245332598686218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1023,0.016634666671355564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,4095,0.027322667340437572
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,2047,0.015504000087579092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,8191,0.03692800054947535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,511,0.014709333578745524
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1023,0.014533333480358124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,4095,0.01825599993268649
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,16383,0.20120000839233398
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,8191,0.023498666783173878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,16383,0.033333333830038704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,32767,0.3869973421096802
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,32767,0.20788266261418661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,1,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1,0.009493333597977957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,3,0.012341332932313284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,3,0.009359999870260557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,7,0.009402666861812273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,7,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,15,0.00949866697192192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,31,0.009701333319147428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,15,0.012266666938861212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,63,0.00980266680320104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,31,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,63,0.012293333808581034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,127,0.010101333260536194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,255,0.014607999473810196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,255,0.014229333649079004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,127,0.012821332861979803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,511,0.019786667078733444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,511,0.0173333336909612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1023,0.02478400121132533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,1023,0.023397333920001984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1,0.013429333766301474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,2047,0.03536533315976461
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,2047,0.03459733227888743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,3,0.013386666774749756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,3,0.01249066616098086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,4095,0.05905599892139435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,4095,0.21724800268809
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,15,0.01340266689658165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,7,0.013418667018413544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,15,0.012527999778588613
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,7,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,31,0.013605333864688873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,31,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,255,0.02181866765022278
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,255,0.01469333345691363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,127,0.014256000518798828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,127,0.013056000073750814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,63,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,63,0.012826666235923767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,511,0.026933332284291584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,4095,0.38065600395202637
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,511,0.017840000490347546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1023,0.02403733382622401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1023,0.036474667489528656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,4095,0.19416000445683798
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1,0.01716800034046173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1,0.014378666877746582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,2047,0.0365280012289683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,2047,0.19917333126068115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,15,0.017535999417304993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,3,0.017221332838137943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,7,0.014501333236694336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,15,0.014357333381970724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,7,0.017429333180189133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,3,0.014490666488806406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,63,0.01785600061217944
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,31,0.01443733274936676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,255,0.023999998966852825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,63,0.015072000523408255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,255,0.01869333287080129
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,127,0.01589866727590561
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,31,0.017557332913080852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,127,0.019130667050679524
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,511,0.03365333378314972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,2047,0.1969333291053772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,511,0.02346666653951009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,2047,0.38869865735371906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,4095,0.7584213415781657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1023,0.20136533180872598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1023,0.03426666557788849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1,0.02402133246262868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,4095,0.374725341796875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,3,0.020634666085243225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,3,0.024656000236670177
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1,0.020319999506076176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,15,0.02462399999300639
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,15,0.02065066620707512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,31,0.025061334172884624
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,7,0.02481599897146225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,31,0.020517333100239437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,63,0.025407999753952026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,7,0.020586666961510975
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,63,0.02093333254257838
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,127,0.027221334477265675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,127,0.022426667312781017
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,255,0.027749332288901012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1023,0.3893119891484578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,255,0.036730666955312095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1023,0.19557867447535196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,511,0.03733866661787033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,511,0.20163732767105103
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,2047,0.7571199735005697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1,0.008767999708652496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,7,0.008874666566650072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,3,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,2047,0.3760106563568115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,4095,1.4959413210550945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,15,0.008746666833758354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,7,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,15,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,4095,0.7325013478597006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,63,0.009077333534757296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,31,0.008869333192706108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,31,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,3,0.008746666833758354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,1,0.011178666104873022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,63,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,127,0.009136000027259191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,511,0.012725333372751871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,255,0.01192533348997434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,255,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,511,0.011877333124478659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1023,0.013280000537633896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,2047,0.014805333067973455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,2047,0.013605333864688873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,4095,0.01783466711640358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,1023,0.012170666207869848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,4095,0.01664000004529953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,8191,0.026906666656335194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,16383,0.03180799881617228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,16383,0.03440533330043157
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,8191,0.022650666534900665
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,32767,0.04164800047874451
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1,0.011946666985750198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,32767,0.0583840012550354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,3,0.012175999581813812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,3,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,7,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,15,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,7,0.011264000087976456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,31,0.012367999802033106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,63,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,15,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,63,0.011141333729028702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,127,0.012639999389648438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,31,0.011663999408483505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,255,0.014933332800865173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,127,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,255,0.012069333344697952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,511,0.01515199989080429
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1023,0.015930666277805965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1023,0.012213333199421564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,511,0.012229333321253458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,4095,0.024533333877722423
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,2047,0.013893333574136099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,4095,0.01685333376129468
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,8191,0.029616000751654308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,2047,0.01758933315674464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,8191,0.022730665902296703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,16383,0.03945599993069967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,16383,0.03492266684770584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,32767,0.058746665716171265
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,32767,0.19916266202926636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,7,0.012602667013804117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,3,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,15,0.012416000167528788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,31,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,15,0.011205332974592844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,7,0.011418666690587997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,31,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,63,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,3,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,63,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,127,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,255,0.015130666395028433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,127,0.01139733319481214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,255,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,511,0.015413332730531693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1023,0.016458666572968166
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,511,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,2047,0.022319999833901722
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1023,0.012741333494583765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,4095,0.027471999327341717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,2047,0.0144213338692983
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,4095,0.017338667064905167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,8191,0.03700799991687139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,8191,0.023743999501069386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,16383,0.2025760014851888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,16383,0.03606399893760681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,32767,0.19164266188939413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,32767,0.38794132073720294
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1,0.013061333447694778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1,0.0122079998254776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,3,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,3,0.01303999995191892
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,7,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,7,0.01309866706530253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,63,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,15,0.01314666618903478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,15,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,31,0.013151999562978745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,127,0.012655999511480331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,255,0.015301333119471868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,63,0.013189333180586496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,127,0.013525333255529404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,255,0.01441066712141037
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1023,0.020938667158285778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,511,0.015930666277805965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,511,0.01470400020480156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,2047,0.0262719988822937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1023,0.01540800059835116
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,4095,0.04013866682847341
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,2047,0.018229333062966663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,4095,0.023391999304294586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,8191,0.2010400096575419
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,8191,0.03385599950949351
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,16383,0.3879306713740031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,16383,0.19325333833694458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,32767,0.3720266819000244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,32767,0.7556213537851969
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1,0.009759999811649323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,1,0.01322666679819425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,3,0.00996800015370051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,7,0.00997866690158844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,3,0.012991999586423239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,7,0.01321600005030632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,15,0.010069333637754122
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,15,0.013082666943470636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,31,0.010138666878143946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,31,0.013269333789745966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,63,0.010330666477481524
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,63,0.013418667018413544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,127,0.011114666859308878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,127,0.013898666948080063
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,255,0.020501332978407543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,255,0.01551466683546702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,511,0.02566933383544286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,511,0.0183999997874101
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1023,0.03541333228349686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,1023,0.024879999458789825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1,0.01770666614174843
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,2047,0.2015999952952067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1,0.015200000256299973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,2047,0.036981334288915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,3,0.01782400036851565
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,3,0.015311999867359797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,7,0.01810666670401891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,7,0.015178666760524115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,15,0.018053332964579265
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,31,0.018272000054518383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,63,0.018378666291634243
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,15,0.015194666882356008
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,63,0.015813333292802174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,127,0.016810666769742966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,127,0.019802667200565338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,31,0.015168000012636185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,255,0.024645333488782246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,255,0.019306667149066925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,511,0.03430933256944021
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,511,0.024405332903067272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1,0.024853333830833435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1,0.021290667355060577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1023,0.03495466709136963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1023,0.1968266765276591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,3,0.025072000920772552
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,7,0.025578667720158894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,3,0.021162666380405426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,7,0.021295999487241108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,2047,0.3799146811167399
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,2047,0.19870400428771973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,15,0.025461333493391674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,31,0.025663999219735462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,31,0.021274665991465252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,63,0.02606400102376938
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,15,0.021189334491888683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,127,0.02293866624434789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,255,0.02812800059715907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,255,0.03750933210055033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,127,0.02775999903678894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,511,0.20438400904337564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,63,0.023418667415777843
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,511,0.03793599953254064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,2047,0.7630453109741211
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,2047,0.3792320092519124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1,0.03886399914820989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1,0.03276266654332479
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1023,0.19834667444229126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1023,0.39370131492614746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,3,0.032416000962257385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,3,0.03932799895604452
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,7,0.03980266551176707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,7,0.032618666688601174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,15,0.03996799886226654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,31,0.03257599969704946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,63,0.03312533348798752
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,63,0.04116799930731455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,127,0.043178667624791466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,127,0.03385599950949351
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,15,0.03271466741959254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,255,0.2042613426844279
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,31,0.04037333279848099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,255,0.04378133515516917
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1023,0.7661279837290446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1023,0.3798186779022217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,2047,1.5062079429626465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,2047,0.7369600137074789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,511,0.19926400979359946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,1,0.013679999858140945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,511,0.3949973185857137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,3,0.013882666826248169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,3,0.016421332955360413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,1,0.01634666696190834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,7,0.013999999811251959
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,7,0.016650666793187458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,15,0.016597333053747814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,31,0.01632000009218852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,63,0.014549333602190018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,15,0.013978666315476099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,63,0.016927999754746754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,31,0.014314666390419006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,127,0.01571200042963028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,255,0.020682666450738907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,127,0.017925333231687546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,255,0.020442667106787365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,1,0.025941332181294758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,511,0.030389333764712017
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,1,0.02266666789849599
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,511,0.025850666066010792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,3,0.026586666703224182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,3,0.022661333282788593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,7,0.026799999177455902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,7,0.022453332940737408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,15,0.026565333207448322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,15,0.022511998812357586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,31,0.027093333502610523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,31,0.022783999641736347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,63,0.027632000545660656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,63,0.023168000082174938
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,127,0.029370665550231934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,127,0.02438933402299881
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,255,0.0390079990029335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,255,0.02961066613594691
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,1,0.04027199993530909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,1,0.034202667574087776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,511,0.20195200045903525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,511,0.039349332451820374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,3,0.04105599969625473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,3,0.03399466723203659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,7,0.041290665666262306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,7,0.034160000582536064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,15,0.03418133407831192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,15,0.04158399999141693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,63,0.04261866708596548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,31,0.04190933207670847
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,63,0.034330666065216064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,31,0.03422933320204417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,127,0.04442666471004486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,127,0.035402665535608925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,1,0.06477333108584087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,255,0.20898133516311646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,255,0.0462719996770223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,1,0.05364799996217092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,3,0.06594133377075195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,511,0.4019786516825358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,511,0.20403732856114706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,7,0.06688533226648967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,3,0.053583999474843345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,15,0.053802669048309326
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,7,0.05365866422653198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,15,0.06683200101057689
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,31,0.06749866902828217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,63,0.0543093333641688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,63,0.06894400219122569
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,127,0.22909865776697794
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,31,0.05414933462937673
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,127,0.056032001972198486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,255,0.4023573398590088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,255,0.20375466346740723
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,511,0.7738773028055826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,1,0.02552533398071925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,511,0.3871519962946574
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,1,0.021850667893886566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,3,0.022085333863894146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,7,0.02550933261712392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,15,0.025573333104451496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,7,0.022143999735514324
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,3,0.02552533398071925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,31,0.025477332373460133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,63,0.02325333406527837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,63,0.026122666895389557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,15,0.022490667800108593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,31,0.02279466638962428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,127,0.025285333395004272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,127,0.027530667682488758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,255,0.0352906659245491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,255,0.03286933402220408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,1,0.04308266441027323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,1,0.03734400123357773
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,3,0.043653334180514015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,3,0.03722133239110311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,7,0.04429866870244344
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,7,0.03709333389997482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,15,0.04463466505209605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,15,0.03724266588687897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,31,0.044794668753941856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,31,0.03758399933576584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,63,0.045696000258127846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,63,0.0378560001651446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,127,0.03851733356714249
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,127,0.04795200129350027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,1,0.06869866450627644
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,255,0.21101866165796915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,255,0.04983466863632202
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,3,0.07026133437951405
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,1,0.05811200042565664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,3,0.05810666580994924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,7,0.07075199981530507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,7,0.058245331048965454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,15,0.07091733316580455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,15,0.05819733440876007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,31,0.07160000006357829
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,31,0.058362667759259544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,63,0.07412800192832947
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,63,0.058789332707722984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,127,0.2188106576601664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,127,0.06113600234190623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,1,0.09734400113423665
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,255,0.41630399227142334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,1,0.11922132968902588
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,255,0.21511467297871908
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,3,0.09745599826176961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,3,0.12154133121172588
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,7,0.12275733550389607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,15,0.1230506698290507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,31,0.12449066837628682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,7,0.09774399797121684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,63,0.21934932470321655
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,63,0.09895466764767964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,15,0.09780800342559814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,31,0.09801600376764934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,1,0.011322667201360067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,7,0.009088000282645226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,3,0.008965333302815756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1,0.008992000172535578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,7,0.011322667201360067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,15,0.008943999807039896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,15,0.011264000087976456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,31,0.009216000015536943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,31,0.011253333340088526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,3,0.011178666104873022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,63,0.009392000113924345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,63,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,127,0.009328000247478485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,127,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,255,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,255,0.012096000214417776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,511,0.01292266696691513
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,511,0.012154666086037954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1023,0.013818666338920593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,1023,0.013770667215188345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,2047,0.015605332950750986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,2047,0.016623999923467636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,4095,0.0227360005180041
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,127,0.4161706765492757
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,4095,0.022511998812357586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,8191,0.02773333340883255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,8191,0.03459733227888743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,16383,0.03741333385308584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,16383,0.058575997749964394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,32767,0.10620799660682678
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,127,0.21640000740687051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,3,0.01250133290886879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1,0.011546666423479715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,3,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,15,0.012741333494583765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,7,0.011178666104873022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,15,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,31,0.012703999876976013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,63,0.012554666648308435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,31,0.011237333218256632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,63,0.011461333682139715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,127,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,127,0.012896000097195307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,255,0.015226667126019796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,511,0.01565333331624667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,255,0.012138667205969492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,511,0.012357333054145178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1023,0.016890666137139004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1023,0.01370666672786077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,32767,0.19875200589497885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,2047,0.022682666778564453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,2047,0.016917333006858826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,4095,0.027717334528764088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,4095,0.022800001005331676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,7,0.012645332763592402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,8191,0.03750933210055033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,255,0.816378672917684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,16383,0.058543999989827476
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,8191,0.03498133271932602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,16383,0.1978506644566854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,32767,0.3794026772181193
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,255,0.4248799880345662
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,3,0.012517333030700684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1,0.011237333218256632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,32767,0.19347200791041055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,3,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,7,0.012650666137536367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,31,0.012863999853531519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,7,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,31,0.011328000575304031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,15,0.012613333761692047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,63,0.012639999389648438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,127,0.013023999830087027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,63,0.011482667177915573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,127,0.01166933278242747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,255,0.015381333728631338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,255,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,511,0.01637866720557213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,511,0.012997332960367203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,2047,0.026367999613285065
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1023,0.021344001094500225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1023,0.0141546664138635
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,4095,0.04048000027736028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,2047,0.017562666287024815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,4095,0.02370133250951767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,8191,0.03579200059175491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,8191,0.20210667451222739
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,16383,0.19241599241892496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,16383,0.3887573480606079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,32767,0.7554559707641602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,32767,0.3677599827448527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,3,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1,0.013306666165590286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,3,0.013306666165590286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,15,0.012874666601419449
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,7,0.012863999853531519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,7,0.013317332913478216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,15,0.013280000537633896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,31,0.01293333371480306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,31,0.013343999783198038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,63,0.012938667088747025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,63,0.013386666774749756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,127,0.013232000172138214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1,0.012778667112191519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,255,0.015978666643301647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,127,0.013872000078360239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,255,0.014682666709025701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,511,0.02080533280968666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1023,0.025775998830795288
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1023,0.01829333355029424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,511,0.015471999843915304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,2047,0.03557866563399633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,15,0.011450666934251785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,2047,0.023541333774725597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,4095,0.03359466542800268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,8191,0.3858933448791504
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,4095,0.22032533089319864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,8191,0.1941173275311788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,16383,0.37141867478688556
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,16383,0.7523466746012369
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,32767,0.7804959615071615
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,32767,1.6693813006083171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,1,0.037776000797748566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,1,0.0321066677570343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,3,0.03809600075085958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,3,0.032485333581765495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,7,0.032074667513370514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,7,0.038575999438762665
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,15,0.03888533264398575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,15,0.032170665760835014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,63,0.04012800008058548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,31,0.03940266619126002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,31,0.03233599911133448
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,63,0.03219733387231827
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,127,0.04237333436806997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,1,0.07425599793593089
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,127,0.033615998923778534
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,1,0.05248000224431356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,3,0.0753119985262553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,7,0.07627733548482259
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,3,0.0524533341328303
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,7,0.05213333169619242
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,15,0.07607999940713246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,15,0.05235200126965841
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,31,0.07711466650168101
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,31,0.05275199810663859
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,63,0.0792746643225352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,63,0.05376000205675761
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,1,0.12664000193277994
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,1,0.09328533212343852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,127,0.23181867599487305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,127,0.05619733532269796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,3,0.1290079951286316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,7,0.13107200463612875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,3,0.0936853289604187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,7,0.09357866644859314
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,15,0.13036266962687174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,15,0.09354133407274882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,31,0.09366400043169658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,31,0.13429333766301474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,63,0.24129066864649454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,1,0.2291253407796224
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,63,0.09516800443331401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,1,0.17233065764109293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,3,0.23384533325831094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,127,0.2203893264134725
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,127,0.47996799151102704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,7,0.23711466789245605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,3,0.17230399449666342
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,7,0.17225066820780435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,15,0.23932266235351562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,15,0.1728000044822693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,31,0.2747573256492615
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,63,0.4813813368479411
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,31,0.17436800400416055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,63,0.2211893399556478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,127,0.902341365814209
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,127,0.4167626698811849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,1,0.06691200037797292
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,1,0.054901331663131714
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,3,0.06743999818960826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,3,0.054986665646235146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,7,0.06849066913127899
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,7,0.05630933245023092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,15,0.06865600248177846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,15,0.055061335364977516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,31,0.07003200054168701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,31,0.05539200206597646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,1,0.13729600111643472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,1,0.09505599737167358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,63,0.05612266560395559
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,3,0.13944000005722046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,63,0.09437867005666097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,3,0.09507200121879578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,7,0.09525866309801738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,15,0.09541333715120952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,15,0.14391466975212097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,31,0.16432533661524454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,7,0.1416106621424357
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,31,0.09578667084376018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,63,0.2796960075696309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,63,0.09864532947540283
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,1,0.2434986631075541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,1,0.1800853411356608
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,3,0.2483946681022644
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,3,0.1802026629447937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,7,0.2563680013020833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,7,0.1803893248240153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,15,0.2773866653442383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,15,0.18054399887720743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,31,0.3158880074818929
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,31,0.18287465969721475
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,63,0.49979201952616376
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,63,0.25085333983103436
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,1,0.44919466972351074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,1,0.3380053440729777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,3,0.4629173278808594
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,3,0.33774932225545246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,7,0.33819735050201416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,7,0.5011039972305298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,1,0.011493333925803503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1,0.009039999917149544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,3,0.009093333035707474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,3,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,7,0.00915733352303505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,7,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,15,0.009072000160813332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,15,0.5164106686909994
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,15,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,31,0.00914666677514712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,15,0.3413333495457967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,63,0.009290666629870733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,63,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,127,0.011605333536863327
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,127,0.009722666814923286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,31,0.5542346636454264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,255,0.012885333349307379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,31,0.01156266654531161
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,255,0.012645332763592402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,511,0.013253333667914072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1023,0.014549333602190018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,1023,0.01710933322707812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,2047,0.020666666328907013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,2047,0.023056000471115112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,4095,0.025765334566434223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,4095,0.034901333351929985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,8191,0.035391998787721
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,8191,0.0588266650835673
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,16383,0.10646933317184448
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,16383,0.1975946625073751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1,0.012661332885424295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,31,0.35862934589385986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,3,0.011525332927703857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,511,0.014085333794355392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,3,0.012671999633312225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,7,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,7,0.011434666812419891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,15,0.012773333738247553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,15,0.01137599969903628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,31,0.011391999820868174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,63,0.012885333349307379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,63,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,127,0.013248000293970108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,127,0.011701333026091257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,255,0.015498666713635126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,255,0.01267733300725619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,511,0.01639466608564059
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,511,0.014298666268587112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1023,0.021642667551835377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1023,0.01716800034046173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,63,0.889520009358724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,4095,0.04088533421357473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,2047,0.023237332701683044
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,2047,0.026975999275843304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,4095,0.03530666728814443
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,31,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,8191,0.1976213256518046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1,0.012730666746695837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1,0.011663999408483505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,16383,0.19210133949915567
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,16383,0.3813333511352539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,3,0.01157333329319954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,7,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,3,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,7,0.011584000041087469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,31,0.012917333592971167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,15,0.011621333658695221
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,63,0.013194666554530462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,31,0.011535999675591787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,15,0.012965332716703415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,63,0.011733333269755045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,127,0.013445333888133367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,127,0.011936000237862269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,255,0.01613333324591319
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,255,0.012981332838535309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,511,0.021055998901526134
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,511,0.01463466634353002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,2047,0.03623466690381368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1023,0.02624000112215678
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,2047,0.023946667710940044
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1023,0.017786666750907898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,4095,0.200762669245402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,4095,0.036570665736993156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,8191,0.387994647026062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,8191,0.1927093267440796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,16383,0.8357333342234293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1,0.012944000462690989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,3,0.013050666699806849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1,0.013642666240533194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,3,0.01357866699496905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,16383,0.36874667803446454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,63,0.45394666989644367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,7,0.012938667088747025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,7,0.013557333499193192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,31,0.01332266628742218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,15,0.013013333082199097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,15,0.013690666606028875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,31,0.013690666606028875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,8191,0.059018666545550026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,63,0.013386666774749756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,63,0.013855999956528345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,127,0.014010666559139887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,127,0.014394666999578476
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1023,0.035504000882307686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,511,0.018581333259741466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1023,0.023786666492621105
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,2047,0.20025600989659628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,511,0.02585600068171819
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,2047,0.03390933324893316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,255,0.015941333025693893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,4095,0.386405348777771
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,4095,0.19427200158437094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,8191,0.7533013025919596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1,0.008885333314538002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,8191,0.3720693190892537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,3,0.011776000261306763
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,3,0.00902399979531765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,16383,1.6696426073710124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,7,0.009178666397929192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,16383,0.7813866933186849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,255,0.02093333254257838
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,7,0.011567999919255575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,15,0.009002666920423508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,15,0.011472000430027643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,31,0.00915733352303505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,31,0.011445333560307821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,63,0.012053333222866058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,127,0.009519999846816063
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,63,0.009242666885256767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,127,0.01163200040658315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,255,0.013194666554530462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,255,0.013295999417702356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,511,0.013989333063364029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,511,0.016303999970356624
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1023,0.019178666174411774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,2047,0.024133334557215374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1023,0.022346665461858112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,2047,0.03422933320204417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,8191,0.19543999433517456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,4095,0.0580213318268458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,8191,0.10578667124112447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,4095,0.03850133220354716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,16383,0.21349332729975382
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1,0.011567999919255575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,16383,0.37987732887268066
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,3,0.01146666705608368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,3,0.012805332740147909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,7,0.012858666479587555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,15,0.012885333349307379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,15,0.011557333171367645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,7,0.011503999431928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,31,0.012986666212479273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,31,0.011551999797423681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,63,0.01301866645614306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,63,0.011616000284751257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,127,0.013541333377361298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,127,0.012096000214417776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,255,0.013386666774749756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,255,0.016384000579516094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,511,0.021290667355060577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,511,0.016522667060295742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1023,0.026250667870044708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1023,0.022618666291236877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,4095,0.058335999647776283
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,4095,0.19660800695419312
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,2047,0.036229332288106285
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,8191,0.3773599863052368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,2047,0.03453333427508672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,8191,0.19246399402618408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1,0.013237333546082178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,16383,0.3731626669565837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,16383,0.7331360181172689
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,3,0.011968000481526056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,3,0.013125333935022354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,7,0.011744000017642975
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1,0.011690666278203329
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,7,0.013242666920026144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,15,0.011605333536863327
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,31,0.01322666679819425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,15,0.013221333424250284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,31,0.011690666278203329
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,63,0.01349866638580958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,63,0.011989332735538483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,127,0.01414399966597557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,127,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,255,0.021055998901526134
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,255,0.01394133393963178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,511,0.017136000096797943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,511,0.025983999172846477
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,2047,0.03555200000603994
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,2047,0.2007253368695577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1023,0.0354720006386439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1023,0.023455999791622162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,4095,0.19183999300003052
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,4095,0.3874186674753825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,8191,0.7535200119018555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,3,0.016719999412695568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,8191,0.368127981821696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,16383,0.7231520016988119
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1,0.013904000322024027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,3,0.013839999834696451
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,16383,1.4888639450073242
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1,0.016623999923467636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,7,0.01687466725707054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,7,0.01381333296497663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,15,0.013845333208640417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,15,0.016890666137139004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,31,0.013786666095256805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,31,0.017055999487638474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,63,0.01717866708834966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,63,0.014490666488806406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,127,0.018458666900793713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,127,0.015402667224407196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,255,0.017903999735911686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,255,0.02345066765944163
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1023,0.1990506649017334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,2047,0.38467200597127277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,511,0.023215999205907185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1023,0.03350399931271871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,511,0.03336533407370249
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,4095,0.3724213441212972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,4095,0.7528906663258871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,2047,0.19447465737660727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1,0.0084906667470932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1,0.010735999792814255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,3,0.00847999999920527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,3,0.010575999816258749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,8191,1.4876213073730469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,8191,0.7253653208414713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,7,0.008623999853928884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,15,0.008629333227872849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,7,0.010693332801262537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,31,0.010645333677530289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,63,0.008842666943868002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,31,0.008736000085870424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,63,0.01062400018175443
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,127,0.009066666786869368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,255,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,511,0.012671999633312225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,255,0.011461333682139715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1023,0.01312000056107839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1023,0.01173866664369901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,511,0.011765333513418833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,2047,0.014282666146755219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,4095,0.016501333564519882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,16383,1.43285338083903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,4095,0.012341332932313284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,8191,0.013562666873137156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,16383,3.3271519343058267
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,8191,0.020960000654061634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,2047,0.011626667032639185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,16383,0.030261332790056866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,32767,0.03296533226966858
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,32767,0.02266666789849599
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1,0.010543999572594961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,16383,0.016741332908471424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,3,0.008634666601816813
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1,0.008549333239595095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,65535,0.033344000577926636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,65535,0.03807466725508372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,3,0.010687999427318573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,7,0.008661333471536636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,15,0.00879466657837232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,7,0.010746666540702185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,63,0.008858666444818178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,15,0.010629333555698395
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,31,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,63,0.01062400018175443
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,127,0.00901333304742972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,31,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,255,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,127,0.010954666882753372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,255,0.011514666179815928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,511,0.012517333030700684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,511,0.011642667154471079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1023,0.01313599944114685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,2047,0.014373333503802618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,4095,0.016656000167131424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1023,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,8191,0.02143999934196472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,2047,0.01198400060335795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,8191,0.013872000078360239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,4095,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,16383,0.01672533278663953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,32767,0.0401653324564298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,65535,0.0498879998922348
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1,0.008703999842206636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,32767,0.02332799881696701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,16383,0.03498666733503342
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,65535,0.03384533276160558
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,3,0.008586666857202848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1,0.010725333044926325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,3,0.010618666807810465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,7,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,15,0.008890666688481966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,31,0.008943999807039896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,7,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,31,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,63,0.009018666421373686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,127,0.009141333401203156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,15,0.010703999549150467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,63,0.01071999967098236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,127,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,255,0.012272000312805176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,511,0.012634667257467905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1023,0.013232000172138214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,2047,0.01451733335852623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1023,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,255,0.011717333147923151
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,511,0.011839999506870905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,2047,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,4095,0.017263999829689663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,4095,0.01309866706530253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,8191,0.0260959987839063
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,32767,0.04115733255942663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,16383,0.017477333545684814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,32767,0.024085332949956257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,8191,0.0145066666106383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,16383,0.031082667410373688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,65535,0.20480000972747803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,65535,0.03487999985615412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1,0.008736000085870424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,15,0.008821333448092142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,3,0.008703999842206636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,7,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,7,0.00878399983048439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,31,0.008874666566650072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,15,0.011264000087976456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,63,0.01121066634853681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,63,0.008858666444818178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,255,0.012250666817029318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,31,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,127,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,127,0.009173333023985228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,511,0.012698666503032049
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,255,0.01211200033624967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1023,0.01341333364446958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,2047,0.015109332899252573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,4095,0.021674667795499165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,511,0.013845333208640417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,8191,0.02699200063943863
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1023,0.013978666315476099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,4095,0.015872000406185787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,2047,0.014720000326633453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,16383,0.03633599976698557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,8191,0.018186666071414948
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,16383,0.023503998915354412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,32767,0.19993066787719727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,65535,0.3901439905166626
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1,0.008725333337982496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1,0.01080000028014183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,32767,0.034128000338872276
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,3,0.008602666358153025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,3,0.010672000547250112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,65535,0.20683199167251587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,7,0.008789333204428354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,7,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,15,0.010672000547250112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,31,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,15,0.008879999940594038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,31,0.010698666175206503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,63,0.008890666688481966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,127,0.009232000137368837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,127,0.010949333508809408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,255,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,511,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,255,0.011621333658695221
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,511,0.011754666765530905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,2047,0.014479999740918478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1023,0.01163200040658315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1023,0.013242666920026144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,2047,0.01198400060335795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,4095,0.016704000532627106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,4095,0.013424000392357508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,8191,0.02162666618824005
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,16383,0.03504000107447306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,16383,0.022485333184401195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,8191,0.016730666160583496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,32767,0.04041599979003271
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,32767,0.03475199888149897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1,0.01192533348997434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1,0.01073066641887029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,3,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,65535,0.05724266668160757
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,65535,0.05026666820049286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,3,0.010714666297038397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,7,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,7,0.010746666540702185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,15,0.012416000167528788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,31,0.012191999703645706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,63,0.012213333199421564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,31,0.010928000013033548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,63,0.010773333410422007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,127,0.01246400053302447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,255,0.014949332922697067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,127,0.011039999624093374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,255,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,511,0.015103999525308609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1023,0.01575999955336253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,511,0.011850666254758835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1023,0.01190399999419848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,2047,0.017157333592573803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,2047,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,4095,0.019744000087181728
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,4095,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,16383,0.033973333736260734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,8191,0.01670933390657107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,16383,0.022778667509555817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,8191,0.02886933336655299
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,32767,0.04376000165939331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,32767,0.03500800083080927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,65535,0.057301332553227745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,65535,0.20186134179433188
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,3,0.012128000458081564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,3,0.010949333508809408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,7,0.012223999947309494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,15,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,7,0.010816000401973724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,31,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,15,0.010879999647537867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,31,0.010821333775917688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,127,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,255,0.014874666929244995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,63,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,127,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,255,0.011706666400035223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,511,0.015210667004187902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,511,0.0120319997270902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1023,0.0161920003592968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,4095,0.024447999894618988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1023,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,2047,0.01250133290886879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,2047,0.017632000148296356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,4095,0.014074667046467463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,8191,0.029530666768550873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,8191,0.017456000049908955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,16383,0.03916800022125244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,16383,0.02366400013367335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,32767,0.2042506734530131
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,32767,0.03633599976698557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1,0.01211200033624967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,3,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,65535,0.19198399782180786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,65535,0.39365867773691815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1,0.011952000359694162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,3,0.011391999820868174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,15,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,7,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,7,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,15,0.011519999553759893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,63,0.012448000411192576
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,31,0.01166933278242747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,31,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,63,0.011493333925803503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,127,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,127,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,255,0.01392000044385592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,255,0.015194666882356008
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,511,0.015413332730531693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1023,0.016549333930015564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,2047,0.022154666483402252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,511,0.014042666802803675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1023,0.014405333747466406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,8191,0.036831999818483986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,2047,0.015365333606799444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,4095,0.027274665733178455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,8191,0.023168000082174938
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,4095,0.018005333840847015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,16383,0.20105600357055664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,16383,0.03346666693687439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,32767,0.38594667116800946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,32767,0.19403199354807535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,65535,0.37062398592631024
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,65535,0.7676160335540771
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1,0.009173333023985228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1,0.011850666254758835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,3,0.009258666386206945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,7,0.009216000015536943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,3,0.012063999970753988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,7,0.011973333855470022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,15,0.009285333255926767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,15,0.011973333855470022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,31,0.011962667107582092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,31,0.009365333244204521
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,63,0.009461333354314169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,63,0.012042666474978128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,127,0.010048000141978264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,127,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,255,0.014111999422311783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,255,0.013760000467300415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,511,0.019013332823912304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,511,0.016949333250522614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1023,0.023130667706330616
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1023,0.024234667420387268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,2047,0.035061334570248924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,2047,0.03393599887688955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,8191,0.1949066718419393
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,8191,0.3800373474756877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,4095,0.05886933207511902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,4095,0.1970720092455546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,3,0.011952000359694162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,7,0.013264000415802002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,7,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1,0.012122667084137598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,3,0.013167999684810638
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,31,0.013061333447694778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,31,0.012042666474978128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,63,0.013584000368913015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,63,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,127,0.01414399966597557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,15,0.013072000195582708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,127,0.012778667112191519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,15,0.012058666596810022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,255,0.021242665747801464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,255,0.014474666366974512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,511,0.026367999613285065
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,511,0.017551999539136887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1023,0.023685333629449207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1023,0.036117332677046456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,4095,0.3789333502451579
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,4095,0.19229867060979208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,8191,0.7327733039855957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,2047,0.19849065939585367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,2047,0.0360000009338061
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1,0.014149333039919535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,3,0.01685333376129468
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1,0.016693333784739178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,8191,0.3712746699651082
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,3,0.014192000031471252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,15,0.0169813334941864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,15,0.014175999909639359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,31,0.016997333616018295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,63,0.017429333180189133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,31,0.01434133326013883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,7,0.016970666746298473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,7,0.014080000420411428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,63,0.014762666076421738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,127,0.018522666146357853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,127,0.015520000209410986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,255,0.023365333676338196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,255,0.01851733277241389
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,2047,0.38710399468739826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,511,0.03315199911594391
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,511,0.023578666150569916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,2047,0.19521600008010864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,4095,0.7564106782277426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1023,0.19966399669647217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1023,0.033488000432650246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,4095,0.3732106685638428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1,0.023887999355793
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1,0.020106667031844456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,3,0.024149333437283833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,7,0.020293333878119785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,15,0.02436800052722295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,8191,0.7276639938354492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,8191,1.4941706657409668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,15,0.02033599962790807
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,31,0.02477866659561793
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,7,0.024298667907714844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,31,0.020319999506076176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,3,0.020106667031844456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,63,0.025045332809289295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,127,0.021920000513394673
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,63,0.02077866718173027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,127,0.026693334182103474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,255,0.0364479993780454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1023,0.3874773184458415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,255,0.0271573339899381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,511,0.20126932859420776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,511,0.03669333209594091
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1023,0.20703999201456705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,2047,0.37544000148773193
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,2047,0.8389653364817301
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,3,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,3,0.008767999708652496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,4095,0.7286400000254313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1,0.0086666668454806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,4095,1.4934560457865398
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,7,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,7,0.00878399983048439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,31,0.008832000195980072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,15,0.008874666566650072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,15,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,31,0.011637333780527115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,63,0.00916800027092298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,127,0.009077333534757296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,63,0.010960000256697336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,127,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,255,0.012538666526476542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,511,0.012639999389648438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,511,0.011786667009194693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,255,0.01181866725285848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1023,0.012085333466529846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1023,0.013151999562978745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,2047,0.014709333578745524
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,2047,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,4095,0.01732800031701724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,4095,0.016496000190575916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,16383,0.03164266546567281
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,8191,0.02250133454799652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,8191,0.02664000044266383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,16383,0.03431999931732813
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,32767,0.04134399940570196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,32767,0.058335999647776283
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1,0.011946666985750198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,65535,0.20150933663050333
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,65535,0.10587199529012044
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,8191,1.4395893414815266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1,0.011039999624093374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,3,0.012223999947309494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,3,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,7,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,8191,2.964821179707845
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,7,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,31,0.012304000556468964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,15,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,31,0.010858666151762009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,63,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,255,0.014858666807413101
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,127,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,63,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,127,0.012543999900420507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,511,0.015189333508412043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,255,0.011882666498422623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,511,0.012058666596810022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1023,0.01598400001724561
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1023,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,4095,0.024373332659403484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,8191,0.029701332251230877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,2047,0.013631999492645264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,2047,0.017583999782800674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,4095,0.01676799977819125
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,8191,0.022842665513356526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,16383,0.03950933367013931
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,16383,0.034714666505654655
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,32767,0.19856532414754233
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,32767,0.058517331878344216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1,0.012213333199421564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,3,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,65535,0.21314134200414023
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,3,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,65535,0.4231040080388387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,7,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,15,0.012250666817029318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,15,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,7,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,31,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,31,0.01110400011142095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,63,0.01246400053302447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,127,0.012554666648308435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,63,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,127,0.011567999919255575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,255,0.014837333311637243
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,255,0.012133333832025528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,511,0.015562667200962702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1023,0.016437333077192307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,511,0.012213333199421564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,2047,0.02229333420594533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1023,0.012666666259368261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,4095,0.02739733209212621
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,8191,0.03686933219432831
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,2047,0.014149333039919535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,4095,0.017674667139848072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,8191,0.023567999402681988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,16383,0.20171199242273966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,16383,0.035930665830771126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1,0.012202666451533636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,32767,0.38707733154296875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,32767,0.1912426749865214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1,0.013061333447694778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,3,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,65535,0.36954665184020996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,3,0.013056000073750814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,7,0.013007999708255133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,65535,0.8377919991811117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,7,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,15,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,15,0.01310933381319046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,63,0.012400000045696894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,31,0.013077333569526672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,127,0.012746666868527731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,127,0.013562666873137156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,63,0.012944000462690989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,255,0.015376000354687372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,511,0.01588800052801768
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,255,0.014384000251690546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,511,0.014677333335081736
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1023,0.021338666478792827
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1023,0.015562667200962702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,2047,0.026021334032217663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,4095,0.040106666584809623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,4095,0.023402666052182514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,8191,0.20043200254440308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,2047,0.018325333793958027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,8191,0.03359466542800268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,16383,0.1930346687634786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,16383,0.428602655728658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,32767,0.3706666628519694
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,32767,0.7537546952565511
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,65535,0.7257493336995443
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,65535,1.672320048014323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1,0.009568000212311745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,3,0.009733333562811216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,3,0.012655999511480331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,7,0.009775999933481216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,7,0.012954667210578918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,15,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,15,0.009839999799927076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,31,0.00985599992175897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,31,0.012773333738247553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,63,0.010154666379094124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,63,0.01301866645614306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,127,0.010714666297038397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,127,0.013301332791646322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,255,0.01498666654030482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,255,0.01977066695690155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,511,0.01811733345190684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,511,0.02462933212518692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1023,0.034330666065216064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1023,0.024405332903067272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1,0.01720533271630605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,2047,0.03660800059636434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,2047,0.21619200706481934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1,0.01462399959564209
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,3,0.014864000181357065
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,3,0.017509333789348602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,4095,0.19553599754969278
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,4095,0.38260265191396076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,15,0.014682666709025701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,15,0.017557332913080852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,7,0.01748266691962878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,31,0.017680000513792038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,7,0.014544000228246054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,31,0.014949332922697067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,127,0.019141333798567455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,127,0.016127999871969223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,255,0.024106666445732117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,255,0.01897066707412402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,511,0.033728001018365227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,511,0.024197332561016083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,63,0.018122666825850803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,63,0.015397333850463232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,4095,0.37480000654856366
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,4095,0.7317333221435547
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1023,0.1950719952583313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1023,0.034287999073664345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1,0.02426133304834366
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1,0.020784000555674236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,3,0.020725333442290623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,3,0.024714666108290356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,2047,0.19711466630299887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,2047,0.37653334935506183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,15,0.02481066683928172
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,7,0.024959998826185863
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,15,0.02090666691462199
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,7,0.020821332931518555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,31,0.025407999753952026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,31,0.020986666282018025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,127,0.027280000348885853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,127,0.022469334304332733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,255,0.03730666637420654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,255,0.027503999571005504
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,63,0.025631998976071674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,63,0.021040000021457672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,511,0.03775999943415324
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,511,0.2023199995358785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,2047,0.7598240375518799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,2047,0.3770773410797119
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1023,0.19658132394154867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,3,0.03893866638342539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,4095,0.7314933141072592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1023,0.39085865020751953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,3,0.03219733387231827
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,4095,1.4985814094543457
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1,0.032186667124430336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1,0.038149334490299225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,7,0.03242666771014532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,15,0.039034667114416756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,31,0.0400693342089653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,31,0.032442666590213776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,63,0.032602667808532715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,63,0.04043200115362803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,7,0.03942399968703588
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,127,0.042965332667032875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,127,0.03317866722742716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,15,0.032357332607110344
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1023,0.763264020284017
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,255,0.04358933369318644
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,255,0.2027733325958252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1023,0.3776266574859619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,2047,0.7847572962443033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,511,0.19646400213241577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,511,0.401253342628479
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,2047,1.5029120445251465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1,0.013536000003417334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,4095,3.3497918446858725
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1,0.015658666690190632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,7,0.013637332866589228
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,4095,1.4440479278564453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,3,0.013461332768201828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,3,0.01573866605758667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,15,0.015626666446526844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,15,0.013749333719412485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,7,0.01573333392540614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,31,0.013797332843144735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,31,0.015669333438078564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,63,0.014277332772811254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,63,0.01621333385507266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,127,0.01720000058412552
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,255,0.019600000232458115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,255,0.020346666375796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,127,0.01552533358335495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,511,0.030447999636332195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,511,0.024832000335057575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1023,0.1943626602490743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1,0.021589333812395733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1,0.02554133286078771
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,3,0.02587199956178665
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1023,0.0352906659245491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,3,0.021722666919231415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,7,0.025983999172846477
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,7,0.02183466653029124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,15,0.021509334444999695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,31,0.026346666117509205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,31,0.021903999149799347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,63,0.021989333132902782
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,15,0.026101333399613697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,127,0.02863999952872594
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,127,0.023525332411130268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,255,0.03817066550254822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,255,0.028607999285062153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,511,0.03819733361403147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,511,0.20975999037424722
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1,0.0394400010506312
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1,0.03314133236805598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1023,0.20129066705703735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1023,0.3812906742095947
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,3,0.04002666721741358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,3,0.033226666351159416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,7,0.03340800106525421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,7,0.04068800061941147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,15,0.04031999905904134
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,31,0.03350399931271871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,63,0.04166933397452036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,63,0.033573334415753685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,31,0.04125333329041799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,15,0.03342399994532267
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,127,0.04401599864164988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,127,0.03421333432197571
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,255,0.20602667331695557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,255,0.04460800190766653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1,0.06377600133419037
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,511,0.39746665954589844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,511,0.20070399840672812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1,0.05294933418432871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,3,0.06513066589832306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,3,0.05253866811593374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1023,0.38416000207265216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1023,0.7953066825866699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,15,0.05282666782538096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,15,0.06613866488138835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,31,0.06704000135262807
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,7,0.06614933411280315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,7,0.0529013325770696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,63,0.053472002347310386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,127,0.05454933146635691
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,127,0.20612800121307373
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,31,0.05303466816743215
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,255,0.3975893259048462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,255,0.20091732343037924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,63,0.06886933247248332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1023,1.5059572855631511
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1023,0.7671626408894857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,511,0.3829973141352336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,511,0.80730668703715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,1,0.02120000123977661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,3,0.021482666333516438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,1,0.02364266663789749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,3,0.023685333629449207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,7,0.021898667017618816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,15,0.023599999646345775
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,31,0.02218666672706604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,31,0.02380266785621643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,63,0.022618666291236877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,7,0.02350933353106181
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,15,0.0216799999276797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,63,0.024122667809327442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,127,0.02454400062561035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,127,0.025573333104451496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,255,0.03438399980465571
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,255,0.031023999055226643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,511,0.19839467604955038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,511,0.04053866614898046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,1,0.04134399940570196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,1,0.03495466709136963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,3,0.042165334026018776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,3,0.03522133330504099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,7,0.042506664991378784
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,7,0.03518400092919668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,15,0.0424586683511734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,15,0.03509333233038584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,31,0.04309333364168803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,31,0.0352906659245491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,63,0.04387199878692627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,63,0.03533866753180822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,127,0.04599999884764353
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,127,0.03615466753641764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,255,0.2048106590906779
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,1,0.06677866478761037
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,1,0.05593066910902659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,255,0.047824000318845115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,511,0.39291731516520184
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,511,0.20877333482106528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,3,0.06806399921576183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,7,0.06902400155862172
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,3,0.05619200070699056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,7,0.05605333546797434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,15,0.0690880020459493
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,31,0.05637866755326589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,15,0.05625066657861074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,31,0.0703359991312027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,63,0.07102933526039124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,63,0.05691199998060862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,127,0.21220799287160239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,127,0.05916266640027364
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,255,0.40772799650828045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,1,0.11752000451087952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,255,0.2093813419342041
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,1,0.09541866183280945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,511,0.3917173147201538
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,511,0.8095466295878092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,3,0.11965866883595784
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,3,0.09541866183280945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,7,0.12141866485277812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,15,0.12142399946848552
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,7,0.09568533301353455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,31,0.09614933530489604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,63,0.21293866634368896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,63,0.0967733363310496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,15,0.09543466567993164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,31,0.12299199899037679
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,127,0.40809067090352374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1,0.00901333304742972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,3,0.009056000038981438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,127,0.21088000138600668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,3,0.01116266722480456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,7,0.009098666409651438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,15,0.009018666421373686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,15,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,31,0.009119999905427298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,31,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1,0.011125333607196808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,63,0.009136000027259191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,63,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,127,0.009423999736706415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,127,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,255,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,255,0.011871999750534693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,7,0.011359999577204386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,511,0.012746666868527731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,511,0.012037333101034164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1023,0.013631999492645264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1023,0.01369599997997284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,2047,0.015274666249752045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,2047,0.016672000288963318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,255,0.7825600306193033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,4095,0.022416000564893086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,4095,0.022613334159056347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,8191,0.027317332724730175
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,8191,0.034703999757766724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,32767,0.1978293259938558
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,16383,0.05834666887919108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,32767,0.10628799597422282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,65535,0.2137226661046346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,65535,0.3831466833750407
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1,0.012293333808581034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,255,0.39449067910512287
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,3,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,3,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,7,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,15,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,15,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,31,0.012400000045696894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,63,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,31,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,63,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,127,0.011488000551859537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,127,0.012778667112191519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,255,0.015013333410024643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,255,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,511,0.01543466622630755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,511,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1023,0.016602666427691776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1023,0.01394133393963178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,2047,0.016906666258970898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,2047,0.022272000710169475
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,4095,0.027600000301996868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,4095,0.02279466638962428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,16383,0.03728000074625015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,8191,0.03728533287843069
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,8191,0.0348693331082662
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,16383,0.19720532496770224
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,7,0.012479999413092932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,16383,0.05878399809201559
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,32767,0.37798933188120526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,511,1.5234932899475098
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,32767,0.19309866428375244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,65535,0.3738400141398112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,3,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,65535,0.7350719769795736
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,3,0.011391999820868174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,7,0.011338666081428528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,7,0.012608000387748083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,15,0.011429333438475927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,15,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,31,0.012538666526476542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,31,0.011359999577204386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,63,0.01257066677014033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,63,0.011445333560307821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,255,0.015263999501864115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,127,0.011626667032639185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,255,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,511,0.01624533285697301
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,511,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1023,0.021269333859284718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,511,0.7526133060455322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,2047,0.026202666262785595
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1023,0.014479999740918478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,2047,0.01747200017174085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,4095,0.04049599915742874
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,4095,0.023754666248957317
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,8191,0.036042665441830955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,16383,0.38763201236724854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,16383,0.19165333112080893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,32767,0.754309336344401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,32767,0.36908264954884845
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,3,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1,0.013280000537633896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,65535,0.721407969792684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,7,0.012554666648308435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,3,0.013440000514189402
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,7,0.013343999783198038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,15,0.01249066616098086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,15,0.013264000415802002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,31,0.012618667135636011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,31,0.01340266689658165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,65535,1.6727466583251953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,63,0.012634667257467905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,63,0.01349866638580958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,127,0.014015999933083853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,255,0.014736000448465347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,511,0.020469332734743755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,511,0.015775999675194424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1023,0.025536000728607178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1023,0.018207999567190807
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,255,0.01573866605758667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,2047,0.03523733218510946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,4095,0.19942933320999146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,2047,0.02367999901374181
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,4095,0.033530667424201965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,8191,0.19387733936309814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,8191,0.38521599769592285
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,16383,0.370959997177124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,16383,0.7528159618377686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,32767,0.7249653339385986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,32767,1.6710880597432454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1,0.011530666301647821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,65535,1.547045389811198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,8191,0.20110932985941568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,65535,3.325770696004232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,1,0.0363520011305809
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,1,0.031285333136717476
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,3,0.036933332681655884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,3,0.03129599988460541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,7,0.031167998909950256
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,7,0.037445334096749626
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,15,0.037434667348861694
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,15,0.03133866687615713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,31,0.03817066550254822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,31,0.03166933357715607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,63,0.03905066599448522
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,63,0.031445334355036415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,127,0.03264000018437704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,127,0.041008000572522484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,255,0.20901334285736084
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,1,0.07060266534487407
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,1,0.051541333397229515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,255,0.043925335009892784
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,3,0.07180266578992207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,3,0.05142933130264282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,7,0.0730560024579366
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,7,0.05144000053405762
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,15,0.07283199826876323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,31,0.07348266740640004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,15,0.051738664507865906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,31,0.0518453319867452
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,63,0.07546666761239369
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,63,0.05226666728655497
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,1,0.12314666310946147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,127,0.21813867489496866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,127,0.05541866521040598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,255,0.4095199902852376
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,255,0.20658665895462036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,1,0.09230933586756389
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,3,0.12525866429011026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,3,0.09240000446637471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,7,0.0926026701927185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,31,0.1293280025323232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,7,0.12771200140317282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,15,0.12683733304341635
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,15,0.09265599648157756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,31,0.0928106705347697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,63,0.22640534241994223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,63,0.0939626693725586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,127,0.4272746642430623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,127,0.21307732661565146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,255,0.8267892996470133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,255,0.40007468064626056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,1,0.17151467005411783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,1,0.22525332371393839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,3,0.23046932617823282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,7,0.23333867390950522
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,3,0.17162134250005087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,15,0.17178134123484293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,31,0.26654932896296185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,15,0.23292799790700278
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,7,0.17139732837677002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,31,0.17314134041468301
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,63,0.4283626476923625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,63,0.23225067059199014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,127,0.8078666528066
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,127,0.4063680171966553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,255,0.7689599990844727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,255,1.5535413424173992
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,1,0.06448000172773997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,1,0.05342400074005127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,3,0.06539733211199443
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,3,0.053472002347310386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,7,0.06619200110435486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,15,0.05364799996217092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,7,0.05330666899681091
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,15,0.06630933284759521
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,31,0.06790933509667714
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,31,0.054058666030565895
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,63,0.06985066831111908
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,63,0.05453333258628845
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,127,0.24751466512680054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,127,0.05746666590372721
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,1,0.1301759978135427
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,1,0.09337066610654195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,3,0.1323199967543284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,3,0.09337066610654195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,7,0.1339413324991862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,7,0.09330667058626811
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,15,0.1338986655076345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,15,0.0934826632340749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,31,0.1437120040257772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,31,0.09405333797136943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,63,0.24854934215545654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,63,0.09483200311660767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,1,0.23621867100397745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,1,0.17814399798711142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,127,0.22588799397150675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,3,0.24055999517440796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,127,0.44436800479888916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,3,0.178330659866333
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,7,0.24489066998163858
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,7,0.17799999316533408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,15,0.25626667340596515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,15,0.17868266503016153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,31,0.28911999861399335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,31,0.180458664894104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,63,0.5024693409601847
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,127,0.8492586612701416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,1,0.4412320057551066
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,127,0.4299360116322835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,1,0.3361920118331909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,3,0.450053334236145
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,63,0.2358400026957194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,3,0.336458683013916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,7,0.4755573272705078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,7,0.3365973234176636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,15,0.49907732009887695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,31,0.528656005859375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,15,0.33859201272328693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,31,0.35205864906311035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1,0.011322667201360067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,3,0.008954666554927826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,7,0.009082666908701261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,7,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1,0.010245333115259806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,63,0.8635520140329996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,15,0.009066666786869368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,15,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,31,0.009114666531483332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,31,0.011237333218256632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,3,0.01139733319481214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,63,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,127,0.00926399976015091
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,127,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,255,0.012709333250919977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,255,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,511,0.013056000073750814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,511,0.013904000322024027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1023,0.014266667266686758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1023,0.016832000265518825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,2047,0.02258133391539256
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,2047,0.020346666375796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,63,0.43722132841746014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,4095,0.025472000241279602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,4095,0.03475199888149897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,8191,0.058559998869895935
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,8191,0.035002666215101876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,16383,0.10637332995732625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,16383,0.19564266999562582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,32767,0.21387734015782675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1,0.011285333583752314
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,3,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,3,0.012650666137536367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,7,0.012602667013804117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,7,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,31,0.012661332885424295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,15,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,31,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,15,0.01267733300725619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,63,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,63,0.011354666203260422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,127,0.01310933381319046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,127,0.011685332904259363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,63,0.009242666885256767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,255,0.012469333906968435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,255,0.015168000012636185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,511,0.01617066686352094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,511,0.014111999422311783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1023,0.02149333308140437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,4095,0.040591999888420105
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1023,0.016986666868130367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,2047,0.02295999974012375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,2047,0.02658133457104365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,4095,0.03505066782236099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,8191,0.19683732589085898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,8191,0.05884799857934316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,16383,0.37934398651123047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,16383,0.1913813352584839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,32767,0.3796159823735555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,32767,0.37810667355855304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,32767,0.812064011891683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,3,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,3,0.012655999511480331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,7,0.012826666235923767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,7,0.011514666179815928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,15,0.011509332805871964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,31,0.012826666235923767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,15,0.012730666746695837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,63,0.012991999586423239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,31,0.011343999455372492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,127,0.013445333888133367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,127,0.011850666254758835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,63,0.011578666667143503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,255,0.015909332782030106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,255,0.012960000584522883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,511,0.02096533278624217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,511,0.014495999862750372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1023,0.026047999660174053
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,2047,0.03573866685231527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1023,0.017504000415404636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,2047,0.023818666736284893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,4095,0.20042133331298828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,4095,0.036271999279658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,8191,0.19162132342656454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,8191,0.38672534624735516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,16383,0.3681120077768962
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1,0.013365333278973898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,32767,0.793509324391683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,16383,0.7529226938883463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,3,0.012944000462690989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,3,0.01332266628742218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,32767,1.6676853497823079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,7,0.012784000486135483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,7,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,15,0.013530666629473368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,15,0.012896000097195307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,31,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,31,0.013386666774749756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,63,0.013712000101804733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,127,0.013749333719412485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,63,0.013264000415802002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,127,0.014208000153303146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,127,1.7338773409525554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,255,0.020501332978407543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,255,0.015829333414634068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1023,0.035445332527160645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,511,0.02571733295917511
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,2047,0.19984533389409384
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1023,0.023765332996845245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,2047,0.03368533402681351
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,511,0.0184906671444575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,4095,0.3856746753056844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,8191,0.3715840180714925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,8191,0.7525333563486735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,16383,1.491871992746989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,16383,0.725050687789917
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1,0.009002666920423508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,3,0.011381333072980246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,3,0.008976000050703684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,32767,1.5454400380452473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,7,0.009056000038981438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,7,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,15,0.009072000160813332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,32767,3.3236001332600913
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,31,0.00915733352303505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,15,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,31,0.01137599969903628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,63,0.009194666519761086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,63,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,127,0.009392000113924345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,255,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,127,0.011594666788975397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,4095,0.19447465737660727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,255,0.013141332815090815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,127,0.8672107060750326
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,511,0.013679999858140945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,511,0.016314666718244553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1023,0.019120000302791595
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1023,0.022122666239738464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,2047,0.024117333193620045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,2047,0.0341386670867602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,8191,0.19596266746520996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,8191,0.10560533404350281
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,16383,0.21261866887410483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,4095,0.057818666100502014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,3,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,3,0.011402666568756104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,4095,0.037989333271980286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,7,0.012965332716703415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1,0.012709333250919977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,31,0.013023999830087027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,15,0.011503999431928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,63,0.013013333082199097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,7,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,63,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,127,0.013274667163689932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,127,0.011898666620254517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,255,0.01607999950647354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,31,0.011328000575304031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,255,0.013445333888133367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,511,0.02102400114138921
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,511,0.01653333380818367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1023,0.02606933315594991
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1023,0.02266666789849599
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,4095,0.19595734278361002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,4095,0.05843733251094818
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,2047,0.035962666074434914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,2047,0.03450666616360346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,8191,0.3760960102081299
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1,0.012938667088747025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,8191,0.1916373372077942
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,16383,0.37062935034434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,16383,0.7313173611958822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,3,0.011578666667143503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,7,0.01301866645614306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,7,0.011642667154471079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,15,0.013050666699806849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1,0.011701333026091257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,15,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,63,0.013269333789745966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,31,0.01163200040658315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,3,0.012794667234023413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,63,0.011866666376590729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,31,0.013359999905029932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,127,0.012266666938861212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,127,0.013936000565687815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,255,0.020853333175182343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,255,0.014015999933083853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,511,0.02595199892918269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,511,0.017125333348910015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,2047,0.20037333170572916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,2047,0.03551466763019562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,4095,0.38541332880655926
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1023,0.023200000325838726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1023,0.03583466758330663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,4095,0.19158933560053507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,8191,0.367850661277771
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,8191,0.7528959910074869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1,0.008549333239595095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,16383,0.37801067034403485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,3,0.008602666358153025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,3,0.010698666175206503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,7,0.00879466657837232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,15,0.0086666668454806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,7,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,15,0.01080000028014183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,31,0.010762666662534079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,63,0.008778666456540426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,16383,1.4883306821187336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,127,0.009029333169261614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,16383,0.7237173716227213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,31,0.008837333569924036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,127,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,255,0.012144000579913458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,511,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1023,0.011850666254758835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,255,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1023,0.013056000073750814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,2047,0.014192000031471252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,511,0.011653333902359009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,2047,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,4095,0.016447999825080235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,4095,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,16383,0.030085332691669464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,32767,0.032698666055997215
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,8191,0.020869334538777668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,16383,0.016800000021855038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,8191,0.01369599997997284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,32767,0.022976001103719074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,65535,0.03794133414824804
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,65535,0.033530667424201965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1,0.008512000242869059
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,7,0.008767999708652496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1,0.010901333143313726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,3,0.008559999987483025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,3,0.010863999525705973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,15,0.008709333216150602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,7,0.010746666540702185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,15,0.010746666540702185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,31,0.008821333448092142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,63,0.008714666590094566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,127,0.009077333534757296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,31,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,255,0.012240000069141388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,127,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,255,0.01173866664369901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,511,0.012538666526476542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,511,0.011839999506870905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,2047,0.014287999520699183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1023,0.01314666618903478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1023,0.011685332904259363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,4095,0.01670933390657107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,2047,0.012080000092585882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,4095,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,8191,0.02143999934196472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,16383,0.03470933437347412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,32767,0.04009599983692169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,8191,0.013839999834696451
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,16383,0.016890666137139004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,32767,0.023285334308942158
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,65535,0.04956266780694326
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,65535,0.033930666744709015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1,0.008645333349704742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,3,0.008714666590094566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,3,0.010858666151762009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,15,0.008842666943868002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,7,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,31,0.008837333569924036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,15,0.010869332899649939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,63,0.009045333291093508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,7,0.00873066671192646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,31,0.010858666151762009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,63,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,255,0.012245333443085352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,127,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,511,0.012517333030700684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,255,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,511,0.011893333246310553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1023,0.01333333303531011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,127,0.009141333401203156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,2047,0.014416000495354334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1023,0.01202133297920227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,4095,0.017322666943073273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,2047,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,4095,0.012752000242471695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,8191,0.026176000634829204
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,16383,0.031045332551002502
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,16383,0.01757866640885671
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,32767,0.04081066697835922
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,8191,0.014794666320085526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,32767,0.02421333392461141
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,65535,0.2042613426844279
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1,0.008592000231146812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,65535,0.03504000107447306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,3,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,3,0.008645333349704742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,7,0.010714666297038397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,7,0.008752000207702318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,15,0.008816000074148178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,31,0.00878399983048439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,15,0.010805333654085795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,63,0.00877333308259646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,31,0.010960000256697336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,127,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,63,0.010821333775917688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,127,0.009103999783595404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,255,0.012752000242471695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,511,0.01259200026591619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,511,0.011717333147923151
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,255,0.011823999385039011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1023,0.013157332936922709
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,2047,0.014368000129858652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1023,0.011685332904259363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,2047,0.012005332857370377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,4095,0.016693333784739178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,4095,0.013503999759753546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,8191,0.021498667697111767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,8191,0.016442666451136272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,16383,0.035088000198205314
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,16383,0.02257599929968516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,32767,0.0401706670721372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,32767,0.03449599941571554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1,0.012128000458081564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,65535,0.05681600173314413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1,0.010858666151762009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,3,0.012133333832025528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,7,0.012245333443085352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,65535,0.049914668003718056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,3,0.01090666651725769
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,7,0.010794666906197866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,31,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,15,0.010794666906197866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,31,0.010656000425418219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,63,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,15,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,127,0.012549333274364471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,63,0.012453333785136541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,255,0.014943999548753103
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,127,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,255,0.0116799995303154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,511,0.015024000157912573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1023,0.015850666910409927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,511,0.011722666521867117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1023,0.01184533288081487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,2047,0.017184000462293625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,4095,0.019648000597953796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,2047,0.012223999947309494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,4095,0.013658666362365087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,8191,0.02888533224662145
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,8191,0.01664000004529953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,16383,0.03380800038576126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,16383,0.02292799949645996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,32767,0.0435146689414978
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,32767,0.03491200009981791
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,65535,0.05740800003210703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,65535,0.20191999276479086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,3,0.012272000312805176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,7,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,3,0.010816000401973724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,15,0.012432000289360682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1,0.012096000214417776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,7,0.010832000523805618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,15,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,63,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,31,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,31,0.012383999923865
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,63,0.011007999380429586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,127,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,127,0.01251199965675672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,255,0.015013333410024643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,255,0.011957333733638128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,511,0.015354666858911514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1023,0.01602666700879733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,2047,0.017504000415404636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1023,0.01210133358836174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,2047,0.012805332740147909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,511,0.012122667084137598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,4095,0.024298667907714844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,4095,0.014314666390419006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,8191,0.029520000020662945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,16383,0.03921066721280416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,8191,0.017263999829689663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,16383,0.02363733450571696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,32767,0.03595199932654699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,32767,0.20402665932973227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,65535,0.1914506753285726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,65535,0.3927733500798543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1,0.009109333157539368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1,0.01173866664369901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,3,0.009039999917149544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,3,0.01191466674208641
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,7,0.011887999872366587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,7,0.009343999748428663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,15,0.009194666519761086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,31,0.009189333145817121
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,15,0.011626667032639185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,31,0.011957333733638128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,63,0.011973333855470022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,63,0.009626666704813639
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,127,0.009797333429257074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,127,0.012063999970753988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,255,0.01381333296497663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,255,0.013722666849692663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,511,0.018810667097568512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1023,0.023893333971500397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,511,0.01682666689157486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1023,0.022554665803909302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,2047,0.033674667278925575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,2047,0.03492266684770584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,8191,0.19450666507085165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1,0.012928000340859095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,4095,0.19644800821940103
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,8191,0.3783573309580485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1,0.01181866725285848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,3,0.01313599944114685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,3,0.012063999970753988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,7,0.013056000073750814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,15,0.01303999995191892
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,15,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,4095,0.05878399809201559
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,7,0.011936000237862269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,31,0.01328533391157786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,31,0.011999999483426413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,63,0.01340266689658165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,63,0.01202133297920227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,127,0.01402666668097178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,255,0.021168000996112823
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,255,0.014240000396966934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,511,0.01722666621208191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,511,0.02611733227968216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,127,0.012650666137536367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1023,0.0359199990828832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1023,0.02346133440732956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,4095,0.37677331765492755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,4095,0.19273066520690918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1,0.01674666628241539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,2047,0.1967946688334147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,2047,0.03589866558710734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1,0.013823999712864557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,3,0.016714667280515034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,8191,0.7324639956156412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,3,0.01414399966597557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,7,0.01704000060757001
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,8191,0.4058826764424642
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,15,0.013850666582584381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,31,0.016970666746298473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,31,0.013914667069911957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,63,0.014490666488806406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,63,0.01739199956258138
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,15,0.0169813334941864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,7,0.014021333307027817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,127,0.015226667126019796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,255,0.023290666441122692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,255,0.01806933308641116
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,127,0.018650667121013004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,2047,0.38583465417226154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,511,0.033258666594823204
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,511,0.023189333577950794
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,2047,0.19510932763417563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1,0.00873066671192646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,4095,0.37348798910776776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1023,0.033488000432650246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1023,0.20003199577331543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,3,0.008725333337982496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,4095,0.7556373278299967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,3,0.011045332998037338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,7,0.008837333569924036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,8191,0.7267839908599854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,31,0.008767999708652496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,63,0.008912000184257826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,31,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,8191,1.6750987370808919
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,127,0.00916800027092298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,63,0.01116266722480456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,127,0.01116266722480456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,255,0.012335999558369318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,255,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,511,0.011823999385039011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,15,0.008879999940594038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,511,0.012586666891972223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1023,0.011989332735538483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1023,0.013327999661366144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,2047,0.014581333845853806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,2047,0.013338666409254074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,4095,0.017258666455745697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,4095,0.016501333564519882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,8191,0.026389333109060924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,8191,0.022495999932289124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,16383,0.03169066707293192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,16383,0.034341332813103996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,32767,0.05811200042565664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,32767,0.0410453329483668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1,0.01210133358836174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,3,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1,0.011050666371981302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,65535,0.10599467158317566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,65535,0.20030399163564047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,3,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,15,0.012106666962305704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,7,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,31,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,31,0.01116266722480456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,15,0.011007999380429586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,63,0.012495999534924826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,7,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,63,0.010928000013033548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,127,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,127,0.011312000453472137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,255,0.014677333335081736
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,511,0.015226667126019796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,255,0.011813333878914515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,511,0.011893333246310553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1023,0.01598400001724561
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1023,0.012202666451533636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,4095,0.024608001112937927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,2047,0.017680000513792038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,4095,0.016735999534527462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,2047,0.01368533323208491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,8191,0.029343999922275543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,16383,0.0391839991013209
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,8191,0.022639999787012737
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,16383,0.03468266626199087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,32767,0.0588266650835673
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,32767,0.1979680061340332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1,0.012250666817029318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,65535,0.3802773157755534
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,65535,0.19169066349665323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1,0.011109333485364914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,3,0.012319999436537424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,7,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,15,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,7,0.011237333218256632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,31,0.012309333930412928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,3,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,31,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,63,0.01249066616098086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,15,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,63,0.011317333827416102
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,127,0.012634667257467905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,127,0.011413333316644033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,255,0.012080000092585882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,511,0.015461333096027374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,511,0.012202666451533636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,255,0.015189333508412043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,2047,0.022122666239738464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1023,0.01659199967980385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,4095,0.02720533311367035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1023,0.012400000045696894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,2047,0.014192000031471252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,8191,0.023584000766277313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,4095,0.017263999829689663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,16383,0.2013333241144816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,16383,0.03602133442958196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,8191,0.03686933219432831
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,32767,0.19140267372131348
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,32767,0.3866186539332072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,65535,0.36693867047627765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,65535,0.7570666472117106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1,0.009642666826645533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1,0.012485332787036896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,3,0.012554666648308435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,7,0.00961599995692571
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,3,0.00972800018886725
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,7,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,15,0.00984533317387104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,15,0.01258133351802826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,31,0.009754666437705358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,31,0.01246400053302447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,63,0.009962666779756546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,63,0.012869333227475485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,127,0.013237333546082178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,127,0.010602666685978571
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,255,0.01923199991385142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,255,0.014848000059525171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,511,0.024149333437283833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,511,0.017898666361967724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1023,0.03375466664632162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1023,0.02430933217207591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1,0.01704000060757001
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1,0.01443733274936676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,2047,0.19698667526245117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,2047,0.036602665980656944
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,3,0.017082666357358296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,3,0.014565333724021912
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,7,0.017360000560681026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,4095,0.19475199778874716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,7,0.014607999473810196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,4095,0.3801120122273763
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,15,0.014378666877746582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,31,0.017637333522240322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,15,0.017258666455745697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,31,0.014730667074521383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,63,0.01509333277742068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,127,0.01571200042963028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,255,0.023989332218964893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,255,0.018800000349680584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,511,0.03365333378314972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,511,0.023962666591008503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,63,0.017968000223239262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,127,0.018976000448067982
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1023,0.033946665624777474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1023,0.19403199354807535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,4095,0.3731199900309245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,2047,0.1967839996019999
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1,0.02422933280467987
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,4095,0.7300159931182861
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,2047,0.4132959842681885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,3,0.02067199970285098
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1,0.02049066623051961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,7,0.024864000578721363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,3,0.02455466737349828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,15,0.02476266771554947
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,15,0.020784000555674236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,31,0.02075200031201045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,63,0.025706666211287182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,63,0.020986666282018025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,127,0.02693866689999898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,127,0.02222399910291036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,7,0.020629333953062694
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,255,0.03743999948104223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,255,0.027424000203609467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,31,0.02517866591612498
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,2047,0.7581653594970703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,511,0.03755733370780945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,511,0.20069867372512817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,2047,0.376693328221639
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,4095,0.7392746607462565
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1023,0.38972798983256024
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1023,0.19534399112065634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,4095,1.4967625935872395
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1,0.013530666629473368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,3,0.015285332997639975
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1,0.01522133375207583
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,3,0.013786666095256805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,7,0.013872000078360239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,15,0.015119999647140503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,31,0.014053333550691605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,31,0.015311999867359797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,15,0.013674666484196981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,63,0.014309333016475042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,7,0.01525866612792015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,63,0.015909332782030106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,127,0.015477333217859268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,127,0.016522667060295742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,255,0.0205226664741834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,255,0.019466667125622433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,511,0.02473066747188568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,511,0.029909332593282063
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1,0.025199999411900837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1023,0.1930720011393229
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1023,0.034858666360378265
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1,0.021402666966120403
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,3,0.025519999365011852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,3,0.021365332106749218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,15,0.025722667574882507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,7,0.021274665991465252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,7,0.02571733295917511
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,31,0.021354667842388153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,63,0.02831999957561493
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,63,0.021850667893886566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,15,0.02120000123977661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,31,0.026074667771657307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,127,0.022976001103719074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,127,0.02804800122976303
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,255,0.03800000001986822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,255,0.028250666956106823
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,511,0.19669334093729654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,511,0.037903999288876854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1,0.03926933308442434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1,0.033013333876927696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1023,0.20083733399709067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1023,0.37841065724690753
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,3,0.03975466638803482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,3,0.0328053335348765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,7,0.040522667268911995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,15,0.04026666780312856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,15,0.03292266776164373
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,7,0.033002667129039764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,63,0.03312533348798752
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,63,0.04147200038035711
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,127,0.033770665526390076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,127,0.04377066592375437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,31,0.032816000282764435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,31,0.04090133309364319
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,255,0.22259199619293213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,255,0.044309332966804504
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,511,0.41517865657806396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,511,0.19862399498621622
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,1,0.020853333175182343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,1,0.02277333289384842
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1023,0.3811146815617879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1023,0.8103893597920736
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,3,0.02128533273935318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,3,0.022778667509555817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,7,0.022842665513356526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,7,0.02142400046189626
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,15,0.021530665457248688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,31,0.022810667753219604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,63,0.022197333474953968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,63,0.023237332701683044
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,127,0.024127999941507976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,127,0.02473066747188568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,15,0.02274133265018463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,31,0.021829334398110706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,255,0.03374933451414108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,255,0.029946667452653248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,511,0.19753599166870117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,1,0.04044266790151596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,1,0.03426666557788849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,511,0.03941333293914795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,3,0.04078399886687597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,3,0.03439466655254364
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,7,0.041850666205088295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,7,0.03432533393303553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,15,0.0415786678592364
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,15,0.034458667039871216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,31,0.04192000130812327
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,63,0.04278933505217234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,63,0.03477866699298223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,31,0.034389334420363106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,127,0.04497066636880239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,127,0.035402665535608925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,255,0.20254933834075928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,255,0.046053335070610046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,1,0.06613866488138835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,1,0.05515733361244202
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,511,0.3886559804280599
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,511,0.20649067560831705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,3,0.05522666871547699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,3,0.06737599770228068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,7,0.06836799780527751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,7,0.05535466472307841
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,15,0.068122665087382
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,15,0.055402666330337524
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,63,0.07072533170382182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,63,0.05598933498064677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,31,0.06967466572920482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,31,0.05532266696294149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1,0.008832000195980072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1,0.01101333275437355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,3,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,3,0.008826666822036108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,7,0.00877333308259646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,15,0.008858666444818178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,127,0.20907733837763467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,31,0.00884799969693025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,31,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,63,0.008976000050703684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,63,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,127,0.0574186642964681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,127,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,255,0.012367999802033106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,255,0.011909333368142446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,511,0.01202133297920227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1023,0.013424000392357508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1023,0.013541333377361298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,2047,0.015392000476519266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,2047,0.016538667182127636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,127,0.00933333362142245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,511,0.012837332983811697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,4095,0.022245332598686218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,4095,0.022469334304332733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,8191,0.027162666122118633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,255,0.20536533991495767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,255,0.4026133219401042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,8191,0.03462400039037069
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,16383,0.0367253323396047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,16383,0.05821866790453593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,32767,0.1972586711247762
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,65535,0.21285333236058554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1,0.012175999581813812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1,0.011045332998037338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,3,0.012293333808581034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,7,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,3,0.011237333218256632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,7,0.010960000256697336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,15,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,15,0.010938666760921478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,31,0.01099733387430509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,63,0.01257066677014033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,32767,0.1062506635983785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,31,0.012416000167528788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,63,0.01099733387430509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,127,0.012682666381200155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,127,0.011306667079528173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,255,0.015210667004187902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,255,0.011930666863918304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,65535,0.38202134768168133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,511,0.015509333461523056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,511,0.012229333321253458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1023,0.016645333419243496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1023,0.01360000049074491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,4095,0.02741333345572154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,2047,0.016778666526079178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,2047,0.022495999932289124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,511,0.3906826575597127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,4095,0.023007998863856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,511,0.8035893440246582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,8191,0.03690666705369949
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,8191,0.03478399912516276
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,16383,0.058864002426465355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,16383,0.19636799891789755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,32767,0.38577067852020264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,65535,0.3707786798477173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,3,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,3,0.01126933346192042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,7,0.011205332974592844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,15,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,15,0.011317333827416102
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,7,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,31,0.012453333785136541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,31,0.011205332974592844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,63,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,63,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,127,0.012954667210578918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,127,0.011776000261306763
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,255,0.01526933287580808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,255,0.012400000045696894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,511,0.012629333883523941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,511,0.016165333489576977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1023,0.021114667256673176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,2047,0.02624000112215678
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1023,0.014170666535695394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,4095,0.04031999905904134
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,2047,0.01749333366751671
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,8191,0.20006932814915976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,4095,0.023685333629449207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,8191,0.035743998984495796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,16383,0.38725332419077557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,16383,0.19143466154734293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,32767,0.3676053285598755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,32767,0.7541333039601644
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,32767,0.19320533672968546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,65535,0.7941813468933105
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,65535,1.4912106196085613
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,65535,0.73526930809021
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,1,0.03602133442958196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,1,0.03072533259789149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,3,0.03654933224121729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,3,0.030666666726271313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,7,0.03070933371782303
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,7,0.037104000647862755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,15,0.03695466617743174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,15,0.030762667457262676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,31,0.03742400060097376
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,31,0.0308693324526151
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,63,0.03825599948565165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,63,0.031023999055226643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,127,0.040421334405740104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,127,0.032074667513370514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,255,0.20454933245976767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,255,0.04303466777006785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,1,0.06899733344713847
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,1,0.05086933573087057
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,3,0.07011199990908305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,3,0.05081599950790405
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,7,0.07111466427644093
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,7,0.050928001602490745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,15,0.05113600194454193
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,15,0.07102933526039124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,31,0.07176533341407776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,31,0.05117866893609365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,63,0.07331199944019318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,63,0.05178666611512502
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,127,0.2109066645304362
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,127,0.054773335655530296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,255,0.43758400281270343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,255,0.20428266127904257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,1,0.12210133671760559
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,3,0.09193600217501323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,7,0.1253706713517507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,15,0.1258026659488678
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,15,0.0923413336277008
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,1,0.09193600217501323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,31,0.1274133324623108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,3,0.1237600048383077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,31,0.09246933460235596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,63,0.21971199909845987
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,7,0.09201066692670186
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,63,0.0936959981918335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,127,0.456224004427592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,127,0.21021332343419394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,255,0.8176799615224203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,255,0.3967466751734416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,1,0.06316799918810527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,1,0.05241066714127859
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,3,0.06409066418806712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,3,0.05273599922657013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,7,0.06523199876149495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,7,0.0525493323802948
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,15,0.06531733274459839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,15,0.052560001611709595
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,31,0.05271466573079427
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,63,0.06748266518115997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,63,0.05402666827042898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,127,0.21836266915003458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,31,0.06614933411280315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,127,0.05683733522891998
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,1,0.12679466605186462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,1,0.09215999643007915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,3,0.12889066338539124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,3,0.09232532978057861
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,7,0.1309760014216105
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,15,0.130703995625178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,7,0.09250133236249287
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,15,0.09261332949002583
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,31,0.13704533378283182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,31,0.09295466542243958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,63,0.23452266057332358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,63,0.09417600433031718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,1,0.23326400915781656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,127,0.4283786614735921
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,1,0.17730667193730673
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,127,0.22073600689570108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,3,0.2377600073814392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,3,0.17722666263580322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,7,0.17753066619237265
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,7,0.24136000871658325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,15,0.24826133251190186
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,31,0.2804159919420878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,15,0.17792532841364542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1,0.008976000050703684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,31,0.1785386602083842
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,3,0.011354666203260422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1,0.01139733319481214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,7,0.008965333302815756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,15,0.009119999905427298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,15,0.011312000453472137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,3,0.00892800030608972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,7,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,31,0.009125333279371262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,31,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,63,0.009077333534757296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,63,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,127,0.009541333342591921
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,127,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,255,0.012661332885424295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,255,0.012159999459981918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,511,0.013173333058754602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,511,0.013738666971524557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1023,0.014303999642531076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1023,0.016677333662907284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,2047,0.02011200040578842
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,2047,0.0227360005180041
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,4095,0.02515733242034912
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,4095,0.03454933315515518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,8191,0.03480000048875809
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,8191,0.05851200222969055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,16383,0.1948266625404358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,63,0.4492373466491699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,16383,0.10602666934331258
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,32767,0.37832534313201904
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,32767,0.21384000778198242
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1,0.012245333443085352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,3,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,3,0.012485332787036896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,7,0.01108266661564509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,63,0.23125867048899332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,15,0.012432000289360682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,15,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,31,0.012608000387748083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,31,0.01110400011142095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,63,0.011338666081428528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,63,0.012693333129088083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,127,0.012965332716703415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,127,0.011407999942700068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,255,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,511,0.013877333452304205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,511,0.016042667130629223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1023,0.02139200021823247
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1023,0.01693333312869072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,7,0.012837332983811697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,2047,0.02274133265018463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,4095,0.040549332896868386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,4095,0.035002666215101876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,2047,0.026661333938439686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,8191,0.19541333119074503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,16383,0.37625598907470703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,8191,0.05855466425418854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,255,0.015423999478419622
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,16383,0.19166932503382364
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1,0.01145600030819575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1,0.01258133351802826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,32767,0.37085334459940594
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,32767,0.7922133604685465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,3,0.012736000120639801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,3,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,7,0.012703999876976013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,31,0.012874666601419449
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,7,0.012485332787036896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,15,0.011551999797423681
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,15,0.012917333592971167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,31,0.011328000575304031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,63,0.012997332960367203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,63,0.011407999942700068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,127,0.013376000026861826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,127,0.011994666109482447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,255,0.015882667154073715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,255,0.01268799975514412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,511,0.014426667243242264
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,511,0.020917333662509918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1023,0.026186667382717133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,2047,0.03570133447647095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1023,0.017269333203633625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,2047,0.02359466751416524
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,4095,0.2002346714337667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,4095,0.03612266729275385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,8191,0.3860906759897868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,8191,0.19167999426523843
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,16383,0.3677653471628825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,16383,0.8348266283671061
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1,0.008789333204428354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,32767,0.7940373420715332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,3,0.008997333546479544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,3,0.011141333729028702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,32767,1.6669227282206218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,127,0.8719786802927653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,7,0.011125333607196808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,15,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,7,0.008853333070874214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,15,0.009093333035707474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,127,0.4238080183664958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,31,0.008997333546479544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,31,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,127,0.009455999980370203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,63,0.008997333546479544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,63,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,255,0.012991999586423239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,255,0.012858666479587555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,511,0.013663999736309052
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1023,0.021882665654023487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,511,0.0161920003592968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,2047,0.02386666586001714
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,2047,0.03373866776625315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,8191,0.10559466481208801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,8191,0.19409600893656412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,4095,0.03811733424663544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,4095,0.05772800246874491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,16383,0.37670401732126874
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,16383,0.21267199516296387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,32767,0.8186079661051432
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,3,0.01116266722480456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,3,0.012693333129088083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,32767,0.4232106606165568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1,0.011338666081428528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,7,0.012965332716703415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,7,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,15,0.012746666868527731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,31,0.012896000097195307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,15,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,31,0.011253333340088526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,63,0.012975999464591345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,63,0.011434666812419891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,127,0.013343999783198038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,127,0.011861333002646765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,255,0.01331199953953425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,255,0.015770666301250458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,511,0.020928000410397846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,511,0.01618133361140887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1023,0.025914666553338368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1023,0.022463999688625336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,127,0.011600000162919363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,4095,0.05811200042565664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,4095,0.195306658744812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1023,0.018944000204404194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,2047,0.0340693344672521
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,2047,0.03570133447647095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,8191,0.3744586706161499
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,8191,0.19134400288263956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,16383,0.3730613390604655
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,16383,0.7705439726511637
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,3,0.013013333082199097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,3,0.011525332927703857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1,0.011461333682139715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,7,0.012858666479587555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,7,0.011567999919255575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1,0.012666666259368261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,32767,1.6153653462727864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,15,0.012906666845083237
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,32767,0.7265439828236898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,15,0.011482667177915573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,31,0.012965332716703415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,31,0.011450666934251785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,63,0.01312000056107839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,127,0.013776000589132309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,63,0.01191466674208641
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,127,0.012213333199421564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,255,0.013738666971524557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,511,0.02566933383544286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,511,0.016757333030303318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1023,0.03535466641187668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,2047,0.21991999944051108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,2047,0.03581333408753077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1023,0.023221333821614582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,255,0.020725333442290623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,4095,0.19076800346374512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,4095,0.3845173517862956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,8191,0.3667573531468709
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1,0.01156266654531161
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,8191,0.7520586649576823
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1,0.013594667116800943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,16383,1.6675732930501301
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,3,0.011695999652147293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,16383,0.7221066951751709
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,3,0.013594667116800943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,32767,1.5757652918497722
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,15,0.01179733375708262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,7,0.013786666095256805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,7,0.011877333124478659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,15,0.013557333499193192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,31,0.011941333611806234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,63,0.012202666451533636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,63,0.014021333307027817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,127,0.013370666652917862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,31,0.01349866638580958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,255,0.01775466650724411
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,127,0.015194666882356008
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,32767,3.2595945994059243
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,255,0.01815466706951459
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,511,0.02294933299223582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1023,0.212826669216156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,511,0.027850667635599773
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1023,0.03313066562016805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,2047,0.41872533162434894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,4095,0.7429386774698893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,4095,0.37056533495585126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,2047,0.19381866852442423
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1,0.008442666381597519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,8191,1.479413350423177
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,8191,0.725056012471517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1,0.010735999792814255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,3,0.008703999842206636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,3,0.010410666465759277
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,7,0.010703999549150467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,7,0.008463999877373377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,15,0.008581333483258883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,15,0.010490667074918747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,31,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,63,0.00879466657837232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,31,0.010437333335479101
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,16383,1.5441439946492512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,16383,3.321920077006022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,63,0.01055466632048289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,127,0.009008000294367472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,127,0.010693332801262537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,255,0.01211200033624967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1023,0.015402667224407196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,255,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1023,0.011690666278203329
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,511,0.011488000551859537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,511,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,2047,0.014032000054915747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,2047,0.011786667009194693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,4095,0.012122667084137598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,4095,0.016106666376193363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,8191,0.02081599955757459
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,16383,0.016677333662907284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,16383,0.029552000264326733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,8191,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,32767,0.03194666653871536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,32767,3.077653249104818
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,65535,0.037802666425704956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,65535,0.03334933271010717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,32767,0.02272533377011617
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1,0.008559999987483025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,131071,0.04748799900213877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,131071,0.0554720014333725
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,7,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,3,0.010480000327030817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,3,0.008778666456540426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,7,0.00873066671192646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,15,0.010672000547250112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,31,0.008709333216150602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,15,0.008565333361426989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,63,0.008922666932145754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,31,0.010591999938090643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,127,0.010581333190202713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,127,0.00901333304742972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,63,0.010575999816258749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,255,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1,0.010901333143313726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,255,0.011445333560307821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,511,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,511,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,32767,6.6459096272786455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1023,0.011733333269755045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1023,0.013023999830087027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,4095,0.016682667036851246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,2047,0.01443733274936676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,4095,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,2047,0.011936000237862269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,8191,0.013834666460752487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,16383,0.016757333030303318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,8191,0.021520001192887623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,32767,0.023007998863856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,32767,0.0397173340121905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,16383,0.033861334125200905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,65535,0.049600000182787575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,65535,0.03367999941110611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,131071,0.22524267435073853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,131071,0.05606933434804281
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,3,0.010586666564146677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1,0.010853332777818045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,3,0.008821333448092142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1,0.008623999853928884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,7,0.008890666688481966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,7,0.010709332923094431
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,15,0.008943999807039896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,15,0.010693332801262537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,31,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,31,0.010741333166758219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,63,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,255,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,63,0.010842667271693548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,127,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,511,0.012479999413092932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,127,0.009082666908701261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,255,0.011589333415031433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,511,0.011642667154471079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1023,0.013162666310866674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1023,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,2047,0.01451733335852623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,2047,0.012250666817029318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,8191,0.025583999852339428
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,4095,0.01717866708834966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,4095,0.012928000340859095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,16383,0.031146667897701263
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,8191,0.014378666877746582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,32767,0.040421334405740104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,16383,0.017504000415404636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,32767,0.023930666347344715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,65535,0.20388267437616983
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,65535,0.03479466587305069
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,131071,0.19157334168752035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,131071,0.3970453341801961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1,0.00761600024998188
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1,0.011114666859308878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,3,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,3,0.007711999739209811
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,7,0.007871999715765318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,7,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,31,0.007903999959429106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,15,0.007765333478649457
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,15,0.011472000430027643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,31,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,63,0.007818666597207388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,255,0.010522666076819101
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,63,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,127,0.01146666705608368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,255,0.012335999558369318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,127,0.008031999692320824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,511,0.014975999792416891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,511,0.013807999591032663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1023,0.024432001014550526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,2047,0.04301333427429199
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,2047,0.014378666877746582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,4095,0.08044800162315369
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,8191,0.15498133500417074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1023,0.014170666535695394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,4095,0.015439999600251516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,8191,0.01848000039656957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,16383,0.0233599990606308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,16383,0.30445865790049237
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,32767,0.033930666744709015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,32767,0.6123520135879517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,65535,0.2071946660677592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1,0.008538666491707167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1,0.010581333190202713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,65535,1.2178346316019695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,131071,0.3704479932785034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,3,0.008586666857202848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,3,0.010666667173306147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,7,0.008767999708652496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,15,0.010527999450763067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,7,0.010474666953086853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,15,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,31,0.008837333569924036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,31,0.010672000547250112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,63,0.008752000207702318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,63,0.010741333166758219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,131071,2.428335984547933
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,255,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,127,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,127,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,255,0.011509332805871964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,511,0.012485332787036896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1023,0.01173866664369901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,511,0.011557333171367645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,2047,0.014394666999578476
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1023,0.013162666310866674
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,2047,0.011866666376590729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,4095,0.016645333419243496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,4095,0.013370666652917862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,8191,0.021551998953024547
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,8191,0.016447999825080235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,32767,0.040106666584809623
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,16383,0.035045333206653595
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,32767,0.03453333427508672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,16383,0.022597332795461018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,65535,0.049695998430252075
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,65535,0.05685866872469584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1,0.012144000579913458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1,0.010682666053374609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,131071,0.10177066922187805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,3,0.010656000425418219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,131071,0.20721065998077393
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,3,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,15,0.01211200033624967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,7,0.010821333775917688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,15,0.01071999967098236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,7,0.012175999581813812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,31,0.01209066684047381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,31,0.010602666685978571
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,63,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,127,0.012538666526476542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,63,0.010805333654085795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,255,0.014757333944241205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,127,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,511,0.014933332800865173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,511,0.011781333635250727
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,255,0.011653333902359009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1023,0.01571200042963028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1023,0.011648000528415045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,2047,0.01695999999841054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,4095,0.019909333437681198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,2047,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,4095,0.013728000223636627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,8191,0.02865600089232127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,8191,0.016677333662907284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,16383,0.033813332517941795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,32767,0.04359466830889384
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,16383,0.02254933367172877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,32767,0.03494933247566223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,65535,0.057215998570124306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,65535,0.20092799266179404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1,0.012047999848922094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,3,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1,0.01073066641887029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,131071,0.19157334168752035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,131071,0.38745065530141193
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,7,0.012367999802033106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,7,0.010693332801262537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,15,0.012256000190973282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,15,0.010741333166758219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,63,0.012485332787036896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,31,0.012175999581813812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,31,0.01102399950226148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,127,0.012554666648308435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,127,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,255,0.014805333067973455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,63,0.010949333508809408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,255,0.011877333124478659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,511,0.015253332753976187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,511,0.01192533348997434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1023,0.015781333049138386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,2047,0.01747200017174085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1023,0.012069333344697952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,4095,0.02437866727511088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,8191,0.029285334050655365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,4095,0.014240000396966934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,2047,0.012800000607967377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,8191,0.01725333308180173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,16383,0.03888533264398575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,16383,0.023541333774725597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,32767,0.20368534326553345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,32767,0.03607466568549474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1,0.00761600024998188
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,65535,0.39238933722178143
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,65535,0.2079360087712606
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1,0.01156266654531161
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,131071,0.3675573269526164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,3,0.007807999849319458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,131071,0.7599413394927979
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,3,0.011306667079528173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,15,0.007840000092983246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,7,0.007936000203092894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,15,0.01109333336353302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,31,0.007776000226537387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,7,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,63,0.008005333443482717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,31,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,63,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,127,0.01221866657336553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,511,0.015066667149464289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,255,0.014058666924635569
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,127,0.0080960001796484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,255,0.01055466632048289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1023,0.024517332514127094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,511,0.014250667144854864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,2047,0.0431573341290156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1023,0.01458666721979777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,4095,0.01800000046690305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,4095,0.08053866525491078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,2047,0.015344000111023584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,8191,0.15529599785804749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,8191,0.023423999547958374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,16383,0.3107466697692871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,16383,0.03333866596221924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,32767,0.193722665309906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,32767,0.6173333326975504
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,65535,0.3717813491821289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,65535,1.22380264600118
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,131071,0.7259519894917806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1,0.009050666665037474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,3,0.01157333329319954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,3,0.00927466650803884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,7,0.009312000125646591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,7,0.011493333925803503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,15,0.00915733352303505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,15,0.011557333171367645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,131071,2.443962732950846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,31,0.009258666386206945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,31,0.011674666156371435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,63,0.00943999985853831
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,63,0.01163200040658315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,127,0.009717333440979322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,127,0.011989332735538483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,255,0.013594667116800943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,511,0.018629333625237148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,511,0.01659199967980385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,255,0.0136266661187013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1023,0.022634667654832203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1023,0.02362666775782903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,2047,0.033146666983763375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,2047,0.03482133398453394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,8191,0.1932906707127889
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,4095,0.19453867276509604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,8191,0.41632533073425293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,4095,0.0584799995024999
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,16383,0.7334613005320231
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1,0.012986666212479273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,3,0.013002666334311167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1,0.012037333101034164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,16383,0.3717386722564697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,7,0.013023999830087027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,15,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,15,0.012997332960367203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,31,0.013088000317414602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,3,0.011823999385039011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,31,0.01180800050497055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,63,0.013290667285521826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,7,0.011690666278203329
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,63,0.011920000116030375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,127,0.014015999933083853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,127,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,255,0.020949333906173706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,255,0.014197333405415217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,511,0.017050666113694508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,511,0.026101333399613697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1023,0.035573333501815796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1023,0.02346666653951009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,4095,0.37538135051727295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,4095,0.19171200195948282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,2047,0.19576533635457358
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,2047,0.035749333600203194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1,0.01393066719174385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,8191,0.36851199467976886
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,8191,0.7311466534932455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,16383,0.7237760225931803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,7,0.01699200024207433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1,0.016688000410795212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,15,0.016837333639462788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,7,0.013909333695967993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,15,0.013834666460752487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,16383,1.443189303080241
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,31,0.017093333105246227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,3,0.016858667135238647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,3,0.013770667215188345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,31,0.013807999591032663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,63,0.017360000560681026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,63,0.014314666390419006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,127,0.018506667266289394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,127,0.015103999525308609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,255,0.02346133440732956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,255,0.018138666947682697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,511,0.0230880007147789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,511,0.033226666351159416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,2047,0.38547734419504803
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,2047,0.19458132982254028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1023,0.19966934124628702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1023,0.033258666594823204
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,4095,0.7552853425343832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,8191,0.7276319662729899
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,4095,0.37161068121592206
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1,0.019946667055288952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1,0.01865600049495697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,3,0.018895999838908512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,7,0.01933866615096728
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,3,0.019679999599854153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,7,0.019871999820073444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,8191,1.4937280019124348
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,15,0.01924266666173935
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,16383,3.3362080256144204
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,15,0.020031999796628952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,31,0.019578666736682255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,31,0.019978666057189304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,63,0.019914666811625164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,63,0.020394666741291683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,16383,1.4398132960001628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,127,0.021776000658671062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,127,0.02160000056028366
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,255,0.026682667434215546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,255,0.03136000037193298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1023,0.1936053236325582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,511,0.03640533238649368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,511,0.19330666462580362
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,2047,0.7464640140533447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,2047,0.37383464972178143
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1023,0.41914133230845135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1,0.008506666868925095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,4095,0.7286933263142904
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,3,0.008629333227872849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,3,0.010805333654085795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,7,0.008639999975760778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,7,0.010805333654085795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,15,0.010832000523805618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,15,0.008762666955590248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,31,0.008682666967312494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,4095,1.6680533091227214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,8191,3.328240076700846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,8191,1.4383412996927898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,31,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,63,0.008837333569924036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,63,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,255,0.01173866664369901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,127,0.009125333279371262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,511,0.012416000167528788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,255,0.012319999436537424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,511,0.011690666278203329
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1023,0.013210666676362356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,2047,0.014522666732470194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1023,0.011930666863918304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,2047,0.013354666531085968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,16383,3.0848159790039062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,4095,0.01635733370979627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,4095,0.017423999806245167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,8191,0.02621866762638092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,16383,0.031194667021433514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,8191,0.022522665560245514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,127,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,32767,0.04089066634575526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,65535,0.19940267006556192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,32767,0.058149332801500954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,16383,0.034458667039871216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,65535,0.10594133536020915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,131071,0.42772801717122394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1,0.01198400060335795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,131071,0.21357333660125732
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,7,0.012223999947309494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,3,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,3,0.01219733307758967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,15,0.012042666474978128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,15,0.010885333021481832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,7,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,31,0.012250666817029318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,31,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,63,0.012421333541472753
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,127,0.012373333175977072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,63,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,127,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,255,0.012042666474978128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,255,0.014981333166360855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,511,0.015157333264748255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1023,0.015930666277805965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1023,0.012154666086037954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,2047,0.017557332913080852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,511,0.011936000237862269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,4095,0.024405332903067272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,2047,0.013674666484196981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,4095,0.016645333419243496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,8191,0.029557332396507263
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,8191,0.022730665902296703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,16383,0.039264000952243805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,16383,0.03495466709136963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,32767,0.19767999649047852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,32767,0.05852800110975901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,65535,0.19183466831843057
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,65535,0.38064531485239667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1,0.012165332833925882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1,0.011109333485364914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,7,0.012319999436537424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,3,0.010885333021481832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,131071,0.7373867034912109
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,131071,0.41575467586517334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,7,0.01118933285276095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,3,0.01219733307758967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,15,0.012495999534924826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,31,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,15,0.011039999624093374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,63,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,31,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,127,0.012671999633312225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,63,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,127,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,255,0.014688000082969666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,255,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,511,0.012335999558369318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1023,0.016506666938463848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,2047,0.02199466774861018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,511,0.015418666104475657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1023,0.012543999900420507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,4095,0.027285332481066387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,2047,0.014293332894643148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,8191,0.036864000062147774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,16383,5.894490559895833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,4095,0.017386666188637417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,16383,0.20077866315841675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,8191,0.023578666150569916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,16383,0.036015999813874565
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,32767,0.19074666500091553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,32767,0.38526399930318195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1,0.00785600021481514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1,0.012975999464591345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,65535,0.83952530225118
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,3,0.007893333211541176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,131071,0.7214879989624023
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,3,0.01321600005030632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,15,0.00797333319981893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,7,0.007786666974425316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,131071,1.4923466046651204
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,7,0.01312000056107839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,15,0.013088000317414602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,31,0.008000000069538752
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,31,0.013050666699806849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,63,0.007871999715765318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,127,0.008261333530147871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,63,0.013034666577974955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,255,0.010586666564146677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,127,0.013503999759753546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,255,0.0143306665122509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,2047,0.04320533573627472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1023,0.024469333390394848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,511,0.015216000378131866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1023,0.01551466683546702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,65535,0.40399467945098877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,511,0.014570667097965876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,2047,0.01814933369557063
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,4095,0.08064533273379008
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,4095,0.023056000471115112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,8191,0.1869600017865499
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,8191,0.03358400116364161
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,16383,0.40679999192555744
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,16383,0.193066676457723
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,32767,0.3715466658274333
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,32767,0.7206400235493978
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,65535,0.7237706979115804
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,65535,1.4326507250467937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1,0.00943999985853831
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1,0.012138667205969492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,3,0.009546666716535887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,131071,1.5442293485005696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,3,0.012080000092585882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,7,0.009621333330869675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,131071,3.2059094111124673
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,7,0.01210133358836174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,15,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,15,0.009568000212311745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,31,0.009658666948477427
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,63,0.009872000043590864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,31,0.01209066684047381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,63,0.012543999900420507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,127,0.012879999975363413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,127,0.010501333822806677
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,255,0.01444799949725469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,255,0.018853332847356796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,511,0.023578666150569916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,511,0.017477333545684814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1023,0.023914667467276256
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1023,0.03325333446264267
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,8191,0.7336053053538004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,2047,0.19421867529551187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,8191,0.3711306651433309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1,0.016832000265518825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,2047,0.03630933413902918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1,0.014192000031471252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,4095,0.37697064876556396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,3,0.014101333916187286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,4095,0.1940106749534607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,3,0.016730666160583496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,7,0.016970666746298473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,15,0.017082666357358296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,7,0.014335999886194864
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,31,0.017162666966517765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,63,0.017370666066805523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,63,0.01481066644191742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,127,0.015696000307798386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,127,0.018687999496857326
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,255,0.0234400009115537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,255,0.018298666924238205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,15,0.014298666268587112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,31,0.014453332871198654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,511,0.03309866786003113
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,511,0.02367466688156128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,4095,0.7735466957092285
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1023,0.033733333150545754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1023,0.19329599539438883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,4095,0.3719786802927653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,2047,0.37347733974456787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1,0.023930666347344715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,2047,0.1952213247617086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1,0.020165332903464634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,3,0.02426133304834366
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,8191,1.4401547114054363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,8191,0.78166397412618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,3,0.020362666497627895
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,7,0.020245333512624104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,15,0.020154666155576706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,31,0.02462933212518692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,31,0.020453333854675293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,63,0.020746666938066483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,7,0.024351999163627625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,127,0.026874666412671406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,127,0.021888000269730885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,63,0.025093334416548412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,15,0.024277334411938984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,255,0.03644266724586487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,255,0.02720533311367035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,2047,0.7558026313781738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,2047,0.3750186761220296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,511,0.2198986609776815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,511,0.036730666955312095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1023,0.1949546734491984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,4095,0.729967991511027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1023,0.3877280155817668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,4095,1.4937334060668945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1,0.03179199993610382
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1,0.03262399882078171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,3,0.031898667414983116
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,7,0.031744000812371574
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,8191,1.5581653912862141
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,8191,3.33841609954834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,15,0.033488000432650246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,3,0.03323733309904734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,15,0.03194666653871536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,31,0.031856000423431396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,31,0.034287999073664345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,7,0.03382933388153712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,63,0.03200533241033554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,127,0.03704000016053518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,127,0.0327360009153684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,63,0.03488533447186152
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,255,0.1945706605911255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,255,0.043103997906049095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1023,0.7510986328125
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1023,0.37467201550801593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,2047,0.730026642481486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,2047,1.4914239247639973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,511,0.19527467091878256
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,511,0.38151999314626056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,4095,1.4396959940592449
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1,0.013317332913478216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1,0.014570667097965876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,4095,2.971712112426758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,8191,3.0894292195638022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,3,0.01339200014869372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,3,0.014848000059525171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,7,0.013653332988421122
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,15,0.013487999637921652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,7,0.014794666320085526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,8191,6.679701487223308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,15,0.01482133318980535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,31,0.014885333677132925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,63,0.014245333770910898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,63,0.015402667224407196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,31,0.013674666484196981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,127,0.015189333508412043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,127,0.016122666498025257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,255,0.019071999937295914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,511,0.029882666965325672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,511,0.02403733382622401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,255,0.020202666521072388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1,0.024773334463437397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1023,0.19200533628463745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1023,0.03428266694148382
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1,0.02092266579469045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,2047,0.19779199361801147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,2047,0.37242666880289715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,3,0.02502399931351344
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,3,0.02092266579469045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,7,0.025263999899228413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,15,0.025221332907676697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,7,0.020810666183630627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,31,0.020917333662509918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,63,0.02604266752799352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,63,0.021312000850836437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,15,0.020853333175182343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,31,0.025637333591779072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,127,0.027722666660944622
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,127,0.022490667800108593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,255,0.037621334195137024
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,255,0.027786667148272198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,511,0.19549334049224854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,511,0.03722666700681051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1,0.03909866760174433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1,0.03245333333810171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1023,0.19917333126068115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1023,0.37620266278584796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,3,0.03952533255020777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,7,0.040063999593257904
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,7,0.03253333270549774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,3,0.03254399945338567
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,2047,0.7720639705657959
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,31,0.0405173326532046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,2047,0.40408531824747723
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,31,0.03250666707754135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,15,0.03997866561015447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,63,0.041034666200478874
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,63,0.032618666688601174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,15,0.03244800120592117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,127,0.033514666060606636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,255,0.0439573327700297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,2047,1.5058666865030925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,511,0.19824532667795816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,511,0.3922453324000041
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,255,0.20230400562286377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,127,0.042954668402671814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,2047,0.7350080013275146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1,0.057722667853037514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1023,0.37939198811848956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,7,0.05192000170548757
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1023,0.8424853483835856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1,0.051962668697039284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,7,0.05964800218741099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,3,0.05180799961090088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,3,0.05902933577696482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,15,0.059605335195859276
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,31,0.060789331793785095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,63,0.05231999854246775
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,63,0.06204266846179962
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,127,0.21553067366282144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,127,0.05429866909980774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,255,0.38395734628041583
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,15,0.05197333296140035
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,31,0.0521919975678126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,255,0.19819732507069907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1023,0.7344319820404053
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1023,1.4889973004659016
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,511,0.37931732336680096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,2047,1.5486879348754883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,511,0.830736001332601
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,2047,2.9599040349324546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1,0.020693333198626835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,3,0.020938667158285778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,3,0.021925332645575207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1,0.021685334543387096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,7,0.021914665897687275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,15,0.021930667261282604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,31,0.02144533395767212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,31,0.021989333132902782
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,63,0.02214933435122172
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,63,0.022042666872342426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,7,0.021386665602525074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,15,0.021216000119845074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,127,0.023685333629449207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,127,0.023589332898457844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,255,0.028901333610216778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,255,0.03379733363787333
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,511,0.19401599963506064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,511,0.03851733356714249
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1,0.03977599988381068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1,0.0334346666932106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1023,0.20085867245992026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1023,0.37587201595306396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,3,0.03345600018898646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,3,0.04031466692686081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,7,0.040805332362651825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,7,0.03346133232116699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,15,0.0407679999868075
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,15,0.03346133232116699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,31,0.04093866546948751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,31,0.033717334270477295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,63,0.03376533339420954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,63,0.042208001017570496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,127,0.04435733457406362
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,127,0.03449599941571554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,255,0.19844265778859457
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,255,0.044794668753941856
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1,0.06531199812889099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,511,0.38362665971120197
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,511,0.2009333372116089
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1,0.05435200035572052
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1023,0.3855946858723958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1023,0.7402400175730387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,3,0.06632533172766368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,3,0.05433600147565206
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,7,0.06729599833488464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,15,0.054469332098960876
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,7,0.05458133419354757
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,15,0.06760533154010773
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,63,0.0692799985408783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,63,0.05500266452630361
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,127,0.20616533358891806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,127,0.05726933479309082
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,31,0.06849066913127899
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,255,0.39795200030008954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,31,0.05469333132108053
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,255,0.20324800411860147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1,0.10921066999435425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,511,0.3851519823074341
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,3,0.11145066221555074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,511,0.7706507047017416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1,0.0937439997990926
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,7,0.1130560040473938
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,3,0.0937600036462148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1023,0.7928106784820557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,15,0.11317333579063416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1023,1.6511359214782715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,31,0.1150933305422465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,31,0.09407466650009155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,63,0.21686400969823202
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,15,0.09365866581598918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,7,0.0937439997990926
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,127,0.3892693519592285
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,255,0.7798453172047933
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1023,3.322271982828776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,255,0.38725332419077557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,127,0.20638400316238403
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1,0.01110400011142095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,7,0.00890666681031386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,7,0.010954666882753372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,15,0.008976000050703684
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1023,1.4537386894226074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,63,0.09520533680915833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,31,0.008965333302815756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,63,0.009077333534757296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,63,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,127,0.009338666374484697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,127,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,255,0.012479999413092932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,255,0.011855999628702799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,511,0.012762666990359625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,3,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,511,0.011941333611806234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1023,0.013530666629473368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1023,0.013584000368913015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,2047,0.016538667182127636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,31,0.01101333275437355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,4095,0.02197333425283432
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,2047,0.015322666615247726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,8191,0.026874666412671406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,4095,0.022469334304332733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,16383,0.03682666768630346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,8191,0.03469333300987879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1,0.008879999940594038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,16383,0.058117335041364036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,32767,0.10590400298436482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,32767,0.1961066722869873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,3,0.008816000074148178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,65535,0.38125332196553546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,65535,0.21338667472203574
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,3,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,3,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1,0.01232533281048139
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,131071,0.4169653256734212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,131071,0.8180747032165527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,7,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,7,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,15,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,31,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,63,0.012479999413092932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,31,0.010938666760921478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,63,0.011258666714032492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,127,0.01267733300725619
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,127,0.011429333438475927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,255,0.014997333288192749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,255,0.011887999872366587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,511,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,511,0.015493333339691162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1023,0.01653333380818367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1023,0.013850666582584381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,2047,0.022266666094462078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,4095,0.02743999908367793
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,2047,0.016885332763195038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,8191,0.03721066564321518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,4095,0.022730665902296703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,8191,0.034661332766215004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,16383,0.19562667608261108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,16383,0.05855466425418854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,32767,0.19161067406336466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,32767,0.37619733810424805
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1,0.012330666184425354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,65535,0.732634703318278
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,65535,0.377616008122762
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1,0.011461333682139715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,3,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,3,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,131071,0.8162666956583658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,7,0.012527999778588613
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,131071,1.4444586435953777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,7,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,31,0.012495999534924826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,15,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,511,1.6697866121927898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,15,0.011194666226704916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,63,0.012586666891972223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,511,0.7434240182240804
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,31,0.011312000453472137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,63,0.011391999820868174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,127,0.012773333738247553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,255,0.015311999867359797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,127,0.0116799995303154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,511,0.01617066686352094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1023,0.02120000123977661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,2047,0.02613866577545802
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1023,0.014192000031471252
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,2047,0.017360000560681026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,8191,0.2005386749903361
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,4095,0.023567999402681988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,4095,0.0403466671705246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,8191,0.03580799947182337
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,16383,0.19137599070866904
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,16383,0.4291253487269084
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,32767,0.36718400319417316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,32767,0.7525333563486735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1,0.008021333565314611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,65535,0.7936480045318604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,65535,1.6705600420633953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1,0.013381333400805792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,7,0.008031999692320824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,3,0.013125333935022354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,3,0.008005333443482717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,131071,1.5723946889241536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,7,0.013349333157142004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,15,0.01320533330241839
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,15,0.007936000203092894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,31,0.008256000156203905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,63,0.008389333263039589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,31,0.013280000537633896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,131071,3.3333921432495117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,63,0.013376000026861826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,127,0.008618666479984919
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,127,0.013829333086808523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,255,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,255,0.014655999839305878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,511,0.01532799998919169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1023,0.025077333052953083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1023,0.018165333817402523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,2047,0.043893332282702126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,511,0.015802666544914246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,4095,0.03321066747109095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,2047,0.02363733450571696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,4095,0.18942399819691977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,8191,0.3674079974492391
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,8191,0.1939679980278015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,16383,0.3717493216196696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,16383,0.726037343343099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,32767,0.7247839768727621
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,32767,1.6340746879577637
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,65535,1.5456639925638835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,511,0.012527999778588613
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,255,0.012432000289360682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,65535,3.2787148157755532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,131071,3.0799681345621743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,131071,6.533215840657552
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,1,0.035360001027584076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,1,0.030218665798505146
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,3,0.035930665830771126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,3,0.030080000559488933
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,7,0.036703998843828835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,7,0.03033066789309184
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,15,0.03664533297220866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,15,0.03047466774781545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,31,0.036874666810035706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,31,0.030378667016824085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,63,0.0305173322558403
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,63,0.03762666632731756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,127,0.04012266546487808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,127,0.031471999982992806
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,255,0.19985065857569376
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,255,0.04279466470082601
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,1,0.06700266897678375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,511,0.38461331526438397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,511,0.19935466845830283
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,1,0.050479998191197716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,3,0.06824533144632976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,3,0.050517335534095764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,7,0.0693280001481374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,15,0.06895466645558675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,7,0.05063466727733612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,31,0.05077333251635233
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,15,0.050517335534095764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,63,0.07150400181611379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,31,0.07017600039641063
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,63,0.05143466591835022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,127,0.20563733577728271
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,127,0.05341866612434387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,1,0.12001066406567891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,255,0.1994826594988505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,255,0.4299893379211426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,1,0.09156800309816997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,511,0.3859359820683797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,511,0.751418670018514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,3,0.1221440037091573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,3,0.09130133191744487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,7,0.12351466218630473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,15,0.09156266848246257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,7,0.09167466560999553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,15,0.12385599811871846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,63,0.21427732706069946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,31,0.12519466876983643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,63,0.09283199906349182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,31,0.09202667077382405
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,127,0.41050132115681964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,127,0.20706667502721152
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,1,0.21411200364430746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,1,0.17015467087427774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,255,0.3910133441289266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,255,0.78548796971639
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,3,0.21847466627756754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,511,1.684245268503825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,511,0.7512106895446777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,3,0.17030400037765503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,7,0.22175467014312744
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,7,0.17036267121632895
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,31,0.24684266249338785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,15,0.17063466707865396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,15,0.22130666176478067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,63,0.20933866500854492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,127,0.7701173623402914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,127,0.39717864990234375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,63,0.3975040117899577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,31,0.17197332779566446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,255,1.6831520398457844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,255,0.7559839884440104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,1,0.0621013343334198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,1,0.05146666864554087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,3,0.06340266764163971
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,3,0.051674668987592064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,511,1.5694133440653484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,511,3.3381118774414062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,15,0.06407466530799866
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,31,0.06486933430035909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,15,0.051594664653142296
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,7,0.06437333424886067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,7,0.051738664507865906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,31,0.052154665191968284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,127,0.2092319925626119
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,63,0.06653333206971486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,63,0.052522664268811546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,127,0.055125330885251365
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,255,0.3967573245366414
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,1,0.12330133716265361
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,255,0.20733332633972168
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,1,0.09146666526794434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,3,0.12520000338554382
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,3,0.09133866429328918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,7,0.12719466288884482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,7,0.09129599730173747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,15,0.09158933162689209
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,15,0.12736533085505167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,31,0.1309706668059031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,31,0.09197866916656494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,63,0.21935999393463135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,63,0.09311466415723164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,127,0.41229331493377686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,127,0.21290133396784464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,1,0.22972800334294638
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,255,0.8208426634470621
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,255,0.3993866840998332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,1,0.17655466000239053
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,3,0.23381332556406656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,3,0.1766293247540792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,7,0.2381760080655416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,7,0.17665066321690878
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,15,0.2413439949353536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,31,0.1776533325513204
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,15,0.17697600523630777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,31,0.26952532927195233
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,63,0.22155199448267618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,63,0.4328960180282593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,127,0.812181313832601
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,127,0.41491198539733887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,1,0.42376001675923664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,255,1.699680010477702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,255,0.8262026309967041
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,1,0.3341919978459676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,3,0.4326293468475342
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,3,0.3340533177057902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,7,0.33444265524546307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,7,0.44228267669677734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,15,0.4695520003636678
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,31,0.3500106732050578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,63,0.8021066983540853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,15,0.3354453245798747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,31,0.4883679946263631
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,3,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,7,0.008992000172535578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,7,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,15,0.00897066667675972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1,0.00891733355820179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,15,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,63,0.4156320095062256
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,31,0.008992000172535578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,31,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,63,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,127,0.00926399976015091
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,3,0.008992000172535578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,127,0.011205332974592844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,255,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,255,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,511,0.012752000242471695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,511,0.013562666873137156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1023,0.016613333175579708
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1023,0.014122666170199713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,2047,0.019904000063737232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,2047,0.022858666876951855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,4095,0.025125332176685333
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,8191,0.034517332911491394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,4095,0.03454933315515518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,16383,0.19432532787322998
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,8191,0.058506667613983154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,16383,0.10608533024787903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,32767,0.37677331765492755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,32767,0.21369065841039023
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,65535,0.735589345296224
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,65535,0.4142826795578003
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,63,0.009039999917149544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1,0.012357333054145178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,3,0.012266666938861212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,3,0.01116266722480456
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,15,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,7,0.012533333152532578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,7,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,15,0.011045332998037338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,31,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,31,0.011253333340088526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,63,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,63,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,127,0.012597333639860153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,127,0.011605333536863327
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,255,0.015168000012636185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,255,0.012170666207869848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,511,0.01603200038274129
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,511,0.013823999712864557
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1023,0.016677333662907284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1023,0.021354667842388153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,2047,0.026389333109060924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,2047,0.022954667607943218
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,4095,0.04029866556326548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,4095,0.034847999612490334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,8191,0.1950826644897461
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,8191,0.05880533158779144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,16383,0.3765173355738322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,16383,0.19182399908701578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,32767,0.3709546724955241
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,32767,0.7324639956156412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,65535,0.8139946460723877
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,3,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,65535,1.6205439567565918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,7,0.012639999389648438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,3,0.011450666934251785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,7,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,15,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,15,0.011418666690587997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,31,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,31,0.012602667013804117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,63,0.011525332927703857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,63,0.012533333152532578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,127,0.011786667009194693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,255,0.01590399940808614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,255,0.012613333761692047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,511,0.02060266708334287
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,511,0.014373333503802618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1023,0.025994665920734406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1023,0.01725333308180173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,2047,0.03549866626660029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,2047,0.023818666736284893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,4095,0.03611200054486593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,4095,0.19940799474716187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,127,1.6939733823140461
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,8191,0.3852533499399821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,8191,0.19141866763432822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,16383,0.7508479754130045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,16383,0.3677706718444824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,127,0.7876213391621908
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,32767,1.6687520345052083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1,0.008005333443482717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,65535,1.5736053784688313
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,3,0.008112000301480293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,65535,3.3292160034179688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,7,0.008229333286484083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,3,0.013471999516089758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,7,0.013264000415802002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,15,0.008192000289758047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,15,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,31,0.008352000266313553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,31,0.013557333499193192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,127,0.009066666786869368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,63,0.013584000368913015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,63,0.008410666758815447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,127,0.014266667266686758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,255,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,255,0.015637333194414776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,511,0.018415999909241993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,511,0.016762666404247284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1023,0.026133333643277485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1023,0.023562667270501454
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,2047,0.19141866763432822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,4095,0.3745280106862386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,2047,0.0335413341720899
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,4095,0.193722665309906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,8191,0.3720959822336833
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,8191,0.7413333257039388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,32767,0.7221279939015707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,16383,0.7249546845753988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1,0.013525333255529404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,16383,1.6612745920817058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1,0.008853333070874214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,3,0.00898133342464765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,32767,1.5469333330790203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,7,0.00898133342464765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,3,0.011039999624093374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,32767,3.325397491455078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,255,1.6030079523722331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,15,0.008896000062425932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,7,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,15,0.011045332998037338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,31,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,31,0.00903466654320558
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,63,0.009119999905427298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,63,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,65535,3.0828641255696616
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,127,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,511,0.013338666409254074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,255,0.012602667013804117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,255,0.013007999708255133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1023,0.018810667097568512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1023,0.021930667261282604
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,511,0.015978666643301647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,65535,6.6374664306640625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,2047,0.023706667125225067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,2047,0.03499733408292135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,4095,0.03774400055408478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,4095,0.05761066575845083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,8191,0.2127679983774821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,16383,0.3733919858932495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,16383,0.2129653294881185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,8191,0.1053493320941925
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,32767,0.7301066716512045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,32767,0.4126826524734497
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,3,0.011306667079528173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,7,0.012330666184425354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,7,0.011125333607196808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,15,0.0124746672809124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,65535,0.8152906894683838
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,65535,1.6214613914489746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,15,0.011253333340088526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,31,0.01258133351802826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,63,0.012693333129088083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,31,0.011231999844312668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,63,0.01129066695769628
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,127,0.011706666400035223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,255,3.356426556905111
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,255,0.015552000453074774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,127,0.013104000439246496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,255,0.013258667041858038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,511,0.016250666230916977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1023,0.025792000194390614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,511,0.02053333322207133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1023,0.022202665607134502
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,2047,0.0344106654326121
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,4095,0.05821333328882853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,8191,0.4145493507385254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,8191,0.1914880077044169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,2047,0.035536001125971474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,16383,0.37001065413157147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,16383,0.7284106413523356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,32767,0.8021386464436849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,32767,1.6148212750752766
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1,0.008186666915814081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1,0.011535999675591787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,3,0.011472000430027643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,3,0.008021333565314611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,7,0.008314666648705801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,7,0.011445333560307821
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,15,0.00810666692753633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,65535,1.6082933743794758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,31,0.008314666648705801
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,15,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,31,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,63,0.008549333239595095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,63,0.011727999895811081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,127,0.00897066667675972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,127,0.012058666596810022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,65535,2.8641440073649087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,255,0.013877333452304205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,255,0.011621333658695221
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,511,0.01640533283352852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,511,0.016762666404247284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,2047,0.03543466577927271
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1023,0.02316266546646754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,2047,0.19178134202957153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,4095,0.37415464719136554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,4095,0.20475733280181885
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,127,0.009589333087205887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,8191,0.8087146282196045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1023,0.02647999922434489
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,16383,0.7208693027496338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,16383,1.657925287882487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1,0.008522666369875273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,8191,0.3673226833343506
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,3,0.010698666175206503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1,0.010768000036478043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,3,0.008586666857202848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,7,0.009397333487868309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,15,0.008736000085870424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,7,0.010735999792814255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,31,0.008709333216150602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,31,0.01073066641887029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,15,0.010640000303586325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,63,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,63,0.008933333059151968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,127,0.009002666920423508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,127,0.010858666151762009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,255,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,255,0.011589333415031433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,511,0.012335999558369318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,511,0.011754666765530905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1023,0.012917333592971167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1023,0.011658667276302973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,2047,0.014090667168299357
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,32767,1.5727252960205078
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,2047,0.011957333733638128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,32767,3.3250719706217446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,4095,0.015978666643301647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,4095,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,8191,0.013717333475748697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,16383,0.01684800038735072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,32767,0.03196266790231069
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,32767,0.022986667851607006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,65535,0.03621333340803782
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,65535,0.03323733309904734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,131071,0.046522667010625206
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,131071,0.05564799904823303
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1,0.008586666857202848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1,0.010735999792814255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,3,0.008592000231146812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,3,0.010768000036478043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,16383,0.029232000311215717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,8191,0.020549333343903225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,15,0.008672000219424566
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,7,0.010698666175206503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,7,0.008698666468262672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,15,0.010693332801262537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,63,0.00873066671192646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,31,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,31,0.00878399983048439
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,63,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,127,0.010826667149861654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,255,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,127,0.008938666433095932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,511,0.012357333054145178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,255,0.011962667107582092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,511,0.0116799995303154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1023,0.012949333836634954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,2047,0.013866666704416275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,2047,0.012122667084137598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,4095,0.01621333385507266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1023,0.011600000162919363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,4095,0.012437333663304647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,8191,0.020469332734743755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,65535,2.843776067097982
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,16383,0.03375466664632162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,8191,0.013946666071812311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,16383,0.016917333006858826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,32767,0.03815466662247976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,65535,0.04840533435344696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,4095,0.19381866852442423
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,65535,0.033674667278925575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,131071,0.20409599939982095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,131071,0.05604266623655955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,3,0.007733333234985669
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1,0.010863999525705973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,3,0.01101333275437355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,65535,6.639135996500651
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1,0.007578666632374127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,7,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,15,0.007834666719039282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,7,0.007786666974425316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,31,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,63,0.007925333455204964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,127,0.008047999814152718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,127,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,255,0.010351999973257383
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,255,0.011578666667143503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,511,0.015040000279744467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1023,0.024346667031447094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,511,0.01181866725285848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1023,0.011994666109482447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,2047,0.042949333786964417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,2047,0.012293333808581034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,4095,0.08044800162315369
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,4095,0.01309866706530253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,8191,0.01444799949725469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,8191,0.1548640032609304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,16383,0.3043786684672038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,16383,0.01758933315674464
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,32767,0.023546665906906128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,32767,0.02405333270629247
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,32767,0.6031359831492106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,65535,0.035061334570248924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1,0.008559999987483025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,65535,1.2174239953358967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1,0.010677333921194077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,131071,0.20891199509302774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,3,0.008736000085870424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,31,0.00766933336853981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,7,0.010656000425418219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,3,0.010666667173306147
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,7,0.008682666967312494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,15,0.010640000303586325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,31,0.008586666857202848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,15,0.008762666955590248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,63,0.008912000184257826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,31,0.010714666297038397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,131071,2.4297706286112466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,127,0.008954666554927826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,63,0.010608000059922537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,255,0.011584000041087469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,511,0.012533333152532578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,255,0.012432000289360682
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,127,0.010709332923094431
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,511,0.011600000162919363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,2047,0.014138666292031607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1023,0.013066666821638743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1023,0.011525332927703857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,4095,0.016832000265518825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,2047,0.011946666985750198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,4095,0.013327999661366144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,8191,0.02164799968401591
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,8191,0.01651200031240781
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,16383,0.03491200009981791
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,16383,0.022991999983787537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,32767,0.03446399917205175
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,65535,0.056890666484832764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1,0.011936000237862269
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,32767,0.0401653324564298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,131071,0.10150399804115295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,131071,0.20566932360331217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,65535,0.04878933231035868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1,0.010735999792814255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,3,0.012128000458081564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,3,0.0106133334338665
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,15,0.012138667205969492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,7,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,7,0.012304000556468964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,31,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,63,0.012138667205969492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,31,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,15,0.010650667051474253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,63,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,127,0.012549333274364471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,255,0.014495999862750372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,127,0.011141333729028702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,511,0.014954666296641031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,255,0.01157333329319954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1023,0.015626666446526844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,511,0.011557333171367645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,2047,0.016751999656359356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1023,0.011898666620254517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,2047,0.011962667107582092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,4095,0.01947733387351036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,4095,0.013429333766301474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,8191,0.028016000986099243
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,8191,0.016677333662907284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,16383,0.03342933456103007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,16383,0.022709332406520844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,32767,0.04299733539422353
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,32767,0.035071998834609985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,65535,0.05719466507434845
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1,0.0075626665105422335
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,131071,0.1922933260599772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,131071,0.38675200939178467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1,0.010901333143313726
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,65535,0.20034666856129965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,3,0.010672000547250112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,7,0.007770666852593422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,3,0.007711999739209811
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,7,0.010773333410422007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,31,0.007813333223263422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,15,0.007829333345095316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,15,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,31,0.010773333410422007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,63,0.00797333319981893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,63,0.010746666540702185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,127,0.008522666369875273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,127,0.011247999966144562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,511,0.015002666662136713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,511,0.01184533288081487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1023,0.024447999894618988
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,255,0.010469333579142889
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,2047,0.0430026650428772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1023,0.012085333466529846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,4095,0.0804319977760315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,2047,0.012479999413092932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,4095,0.013973332941532135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,8191,0.1551466683546702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,8191,0.01725333308180173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,16383,0.023418667415777843
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,16383,0.30446932713190716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,32767,0.03608000030120214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,32767,0.6127839883168539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,65535,0.19101866086324057
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,131071,0.368341326713562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,65535,1.2187679608662922
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1,0.009039999917149544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,3,0.009066666786869368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,131071,2.430778662363688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,3,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,7,0.009183999771873156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,7,0.011381333072980246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,15,0.008992000172535578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,15,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,31,0.009136000027259191
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,31,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,63,0.009349333122372627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,63,0.011365332951148352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,127,0.00960533320903778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,127,0.011733333269755045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,255,0.013349333157142004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,255,0.013343999783198038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,511,0.018351999421914417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,511,0.01626666635274887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1023,0.02319466571013133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1023,0.022469334304332733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,2047,0.03301866600910822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,2047,0.03435733417669932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,8191,0.37484800815582275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,8191,0.1934666633605957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,4095,0.058133333921432495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,4095,0.1928960084915161
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1,0.011605333536863327
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1,0.012671999633312225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,16383,0.8171199957529703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,16383,0.371018648147583
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,7,0.012890666723251343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,7,0.011407999942700068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,15,0.011626667032639185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,3,0.011418666690587997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,32767,0.8143626848856608
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,3,0.012709333250919977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,15,0.012837332983811697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,31,0.012960000584522883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,32767,1.4456000328063965
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,31,0.011472000430027643
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,63,0.013301332791646322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,63,0.011744000017642975
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,127,0.012106666962305704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,255,0.020560000091791153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,255,0.013888000200192133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,511,0.016890666137139004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,511,0.025765334566434223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,4095,0.3736000061035156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1023,0.035445332527160645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1023,0.02316266546646754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,127,0.013829333086808523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,4095,0.19128000736236572
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,2047,0.03540800015131632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,8191,0.7285172939300537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,2047,0.21475734313329062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,8191,0.36849065621693927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1,0.011813333878914515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,16383,0.7220586935679117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,3,0.011823999385039011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1,0.013717333475748697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,16383,1.4391039212544758
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,3,0.01369599997997284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,7,0.013728000223636627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,31,0.012191999703645706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,15,0.011930666863918304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,15,0.01369599997997284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,32767,3.2207040786743164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,7,0.01198400060335795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,31,0.013584000368913015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,63,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,63,0.014202666779359182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,127,0.013525333255529404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,255,0.0179626668492953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,255,0.018325333793958027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,511,0.028010666370391846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,511,0.022805333137512207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,127,0.014858666807413101
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,2047,0.37772266070048016
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,32767,1.438154697418213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,2047,0.1939093271891276
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1023,0.1929759979248047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1023,0.03299200038115183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,4095,0.7474986712137858
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,4095,0.3987040122350057
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1,0.00850133349498113
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,8191,0.7800532976786295
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,8191,1.4977493286132812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,3,0.008623999853928884
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,3,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,7,0.010714666297038397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,7,0.00867733359336853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,15,0.009008000294367472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,15,0.010928000013033548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,31,0.008778666456540426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,63,0.010821333775917688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,31,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,63,0.008863999818762144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,16383,1.5474400520324707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,127,0.009072000160813332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,255,0.012410666793584824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,16383,3.332736015319824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,511,0.011616000284751257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1023,0.013397333522637686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1023,0.01209066684047381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,2047,0.014479999740918478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,4095,0.01724799970785777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,4095,0.01646399994691213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,8191,0.026330667237440746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,8191,0.022639999787012737
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,16383,0.03126933425664902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,32767,0.041008000572522484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,511,0.012805332740147909
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,65535,0.10599999626477559
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,16383,0.03446399917205175
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,32767,0.05840533475081126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,65535,0.19868266582489014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1,0.011226666470368704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,131071,0.21296000480651855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,2047,0.01350933313369751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,3,0.012522666404644648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,3,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,131071,0.42836801211039227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,7,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,7,0.012234666695197424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,15,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,31,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,15,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,63,0.011087999989589056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,31,0.01139733319481214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,63,0.012421333541472753
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,127,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,127,0.012639999389648438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,255,0.011770666887362799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,255,0.014954666296641031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,511,0.015568000574906668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,511,0.012037333101034164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1023,0.016085332880417507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1023,0.012282667060693106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,2047,0.01743999992807706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,4095,0.016794666647911072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,4095,0.024512000381946564
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,2047,0.013647999614477158
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,8191,0.029301332930723827
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,8191,0.02292799949645996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,16383,0.03885333240032196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,16383,0.03481066723664602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,32767,0.05866666634877523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,32767,0.2177013357480367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,65535,0.19148266315460205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,65535,0.3797973394393921
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1,0.010954666882753372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,3,0.007717333113153775
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1,0.007754666730761528
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,131071,0.3713013331095378
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,131071,0.7960533301035563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,7,0.008592000231146812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,3,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,7,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,31,0.007903999959429106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,15,0.011136000355084738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,31,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,63,0.008122666428486506
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,15,0.008042666440208754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,63,0.01101333275437355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,127,0.011381333072980246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,255,0.010565333068370819
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,127,0.008287999778985977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,511,0.015135999768972397
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,32767,2.8588425318400064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,255,0.012335999558369318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,511,0.012144000579913458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1023,0.024490666886170704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,2047,0.04306666553020477
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1023,0.012639999389648438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,8191,0.023445333043734234
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,4095,0.017397332936525345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,8191,0.1553759972254435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,4095,0.08041599889596303
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,16383,0.3109653393427531
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,16383,0.03585600107908249
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,32767,0.19076800346374512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,32767,0.615066647529602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,65535,0.3669439951578776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,65535,1.2258079846700032
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,131071,0.7935893535614014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,32767,5.910709381103516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1,0.009408000235756239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1,0.012015999605258306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,131071,2.4448800086975098
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,2047,0.014442666123310724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,3,0.009503999724984169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,3,0.011770666887362799
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,7,0.011839999506870905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,15,0.009679999823371569
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,15,0.011733333269755045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,31,0.009599999835093817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,31,0.013264000415802002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,63,0.009695999945203463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,63,0.012042666474978128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,127,0.010378666842977205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,127,0.012506666282812754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,255,0.018464000274737675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,255,0.014181333283583323
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,511,0.017125333348910015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,511,0.0234400009115537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,7,0.009514666472872099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1023,0.03324266771475474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1023,0.0236160010099411
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,8191,0.4052319924036662
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,2047,0.19315199057261148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,2047,0.03610666592915853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,8191,0.7288426558176676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,4095,0.3738826513290405
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1,0.01659199967980385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,4095,0.19291732708613077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1,0.013872000078360239
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,3,0.013781332721312841
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,16383,1.4453280766805012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,3,0.016869333883126576
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,7,0.013962666193644205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,15,0.013882666826248169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,7,0.016943999876578648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,31,0.017024000485738117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,31,0.014159999787807465
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,63,0.017456000049908955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,63,0.015178666760524115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,15,0.016773333152135212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,127,0.018485333770513535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,127,0.015333333363135656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,255,0.02349333216746648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,255,0.018250666558742523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,511,0.03320533285538355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,511,0.0232640008131663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,16383,0.7274453639984131
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1023,0.1923146645228068
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,4095,0.726032018661499
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1023,0.033904001116752625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,4095,0.3713066577911377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,8191,0.78001602490743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,2047,0.3714933395385742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,2047,0.19401599963506064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,8191,1.4372053146362305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1,0.019978666057189304
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1,0.018874666343132656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,3,0.019226666539907455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,7,0.019952000429232914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,15,0.019530666371186573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,3,0.01995733380317688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,16383,1.5460426012674968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,15,0.019871999820073444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,31,0.019685332973798115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,7,0.019637333850065868
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,63,0.02006400004029274
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,31,0.020069333414236706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,127,0.021759999295075733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,127,0.021754667162895203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,255,0.026602665583292644
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,63,0.020501332978407543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,16383,2.853578567504883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,255,0.03158933420976003
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,511,0.03644266724586487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,511,0.1936053236325582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,2047,0.7474559942881266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1023,0.3925066788991292
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1023,0.19394133488337198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,2047,0.37325334548950195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,4095,1.4849279721577961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,4095,0.7291626930236816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,8191,1.5556853612263997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1,0.014149333039919535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,3,0.013349333157142004
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1,0.012997332960367203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,3,0.014245333770910898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,7,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,8191,2.9606399536132812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,7,0.014085333794355392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,15,0.013376000026861826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,15,0.014256000518798828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,31,0.013594667116800943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,31,0.014165333161751429
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,63,0.014607999473810196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,127,0.014933332800865173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,127,0.015477333217859268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,63,0.01394133393963178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,255,0.019968000551064808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,255,0.018474667022625606
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,511,0.029338667790095013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,511,0.0235359991590182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,16383,2.8566932678222656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1023,0.20990399519602457
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,16383,5.904266357421875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1023,0.0337119996547699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,2047,0.3697173198064168
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,2047,0.19556800524393717
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1,0.020197333147128422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,3,0.02033599962790807
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1,0.02404800057411194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,4095,0.7496053377787272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,4095,0.3747680187225342
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,15,0.02462933212518692
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,15,0.020293333878119785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,7,0.02025066688656807
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,7,0.02481066683928172
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,3,0.024517332514127094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,31,0.025061334172884624
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,63,0.02065066620707512
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,127,0.02699200063943863
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,127,0.02214933435122172
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,255,0.03703466554482778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,255,0.027248000105222065
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,511,0.19324799378712973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,511,0.03696533292531967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,31,0.020501332978407543
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,63,0.025493333737055462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1023,0.3736533323923747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1023,0.19754133621851602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1,0.03297599901755651
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,4095,1.4364533424377441
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,4095,0.7308213710784912
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1,0.03190399954716364
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,7,0.0340639998515447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,2047,0.37752532958984375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,2047,0.7877919673919678
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,7,0.031712000568707786
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,3,0.03377600014209747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,15,0.031898667414983116
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,3,0.03197866678237915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,15,0.034058667719364166
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,31,0.034634667138258614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,63,0.03203733265399933
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,127,0.03753600021203359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,127,0.03291733314593633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,255,0.19477333625157675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,255,0.04307200014591217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,511,0.38204801082611084
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,2047,0.7317279974619547
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,31,0.03221333275238673
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,511,0.19625065724054971
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,2047,1.4938987096150715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,63,0.03542399903138479
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,4095,1.556506633758545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1023,0.37677868207295734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1023,0.8210933208465576
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1,0.02070933332045873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,3,0.021151999632517498
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,3,0.021013334393501282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1,0.021029333273569744
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,4095,2.972192128499349
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,7,0.021231998999913532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,15,0.021087999145189922
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,31,0.02149333308140437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,31,0.021136000752449036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,7,0.02109333376089732
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,15,0.021402666966120403
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,63,0.02125866711139679
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,63,0.02199999988079071
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,127,0.023765332996845245
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,127,0.022831998765468597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,255,0.03369066615899404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,255,0.02788266787926356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,511,0.03765333443880081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,511,0.1918720006942749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1023,0.411135991414388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1023,0.19762667020161948
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1,0.03898133337497711
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1,0.03257599969704946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,3,0.0395359992980957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,3,0.032746667663256325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,2047,0.3775999943415324
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,2047,0.772437334060669
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,7,0.03997866561015447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,7,0.032560000816980995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,15,0.04015466570854187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,15,0.032602667808532715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,31,0.04050133377313614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,31,0.03283733377854029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,127,0.043493335445721946
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,127,0.03367999941110611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,255,0.1956640084584554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,255,0.044405331214269005
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,63,0.04127999891837438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,63,0.03294399877389272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,511,0.3790293137232463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,511,0.19895466168721518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1,0.05925333499908447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1,0.053717335065205894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,3,0.060378665725390114
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1023,0.38175467650095624
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1023,0.7771466573079427
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,7,0.06116800010204315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,3,0.05359466870625814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,7,0.05359466870625814
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,15,0.0610453337430954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,2047,0.7907466888427734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,2047,1.614192008972168
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,31,0.053770666321118675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,63,0.06384533147017162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,63,0.05435733497142792
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,15,0.05372266471385956
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,31,0.06227200229962667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,127,0.19754666090011597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,255,0.2000160018603007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,511,0.38093864917755127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,511,0.7979839642842611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,2047,2.9625279108683267
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,255,0.3851253191630046
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,127,0.05648533503214518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,3,0.008890666688481966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,7,0.010954666882753372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,15,0.009018666421373686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,7,0.008805333326260248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,31,0.00892800030608972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,3,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,31,0.010949333508809408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,63,0.00877333308259646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,63,0.01098666712641716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,127,0.009296000003814697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,2047,1.4495946566263835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,127,0.011071999867757162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,255,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,511,0.012026666353146235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1023,0.013408000270525614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1023,0.013418667018413544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,2047,0.01533866673707962
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,2047,0.016437333077192307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,4095,0.021925332645575207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,8191,0.026842666169007618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,4095,0.022218666970729828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,8191,0.03435733417669932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,16383,0.03653866549332937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,511,0.012858666479587555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1,0.008922666932145754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,16383,0.05821866790453593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,32767,0.19485867023468018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,32767,0.10588799913724263
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1,0.011050666371981302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,65535,0.21312000354131064
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1,0.01219733307758967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,65535,0.4214133421579997
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,3,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,3,0.01101333275437355
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,131071,0.41630399227142334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,131071,0.8161386648813883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,7,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,15,0.012442667037248611
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,7,0.01102399950226148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,15,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,31,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,63,0.012341332932313284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,63,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,31,0.012245333443085352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,127,0.012752000242471695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,127,0.011178666104873022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,255,0.015205333630243937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,255,0.012026666353146235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,511,0.01551466683546702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,511,0.012106666962305704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1023,0.016501333564519882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1023,0.01369599997997284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,2047,0.022341333329677582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,2047,0.01674666628241539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,4095,0.027237333357334137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,4095,0.0225600004196167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,8191,0.03701333453257879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,8191,0.03475733349720637
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,16383,0.05844266712665558
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,16383,0.19510932763417563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,32767,0.19210133949915567
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,32767,0.37402665615081787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1,0.007727999861041705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,65535,0.3710666497548421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,65535,0.7314559618631998
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,3,0.007797333101431529
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,3,0.01110400011142095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1023,0.7886506716410319
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,131071,0.8154933452606201
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,7,0.007983999947706858
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,7,0.01119999960064888
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,15,0.007983999947706858
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,131071,1.6200052897135417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,31,0.00797333319981893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,15,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1023,1.665237267812093
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,63,0.00810666692753633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,63,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,127,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,127,0.008245333408315977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,255,0.012261333564917246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,511,0.01257066677014033
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,255,0.010634666929642359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,2047,0.04331733286380768
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1023,0.02462399999300639
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1023,0.014096000542243322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,2047,0.017322666943073273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,4095,0.023381332556406658
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,4095,0.08076266447703044
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,8191,0.18649067481358847
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,8191,0.03566933423280716
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,16383,0.3633546829223633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,16383,0.1909866730372111
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,32767,0.36713067690531415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,32767,0.7172053654988607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,65535,0.7936800320943197
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,65535,1.6086880366007488
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,131071,1.5712693532307942
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,131071,3.182138760884603
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,31,0.011274666835864386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,511,0.015439999600251516
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1,0.035114665826161705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1,0.029861333469549816
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,3,0.03570133447647095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,3,0.029765332738558452
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,7,0.03623999903599421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,7,0.02995733420054118
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,15,0.03581333408753077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,15,0.029951999584833782
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,31,0.036805334190527596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,31,0.029919999341169994
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,63,0.037434667348861694
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,63,0.030053332448005676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,127,0.03940266619126002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,127,0.031152000029881794
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,255,0.19429866472880045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,255,0.04159466673930486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,511,0.37797868251800537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,511,0.19581866264343262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1,0.06565866867701213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1,0.05013333261013031
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1023,0.7764106591542562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1023,0.3784586588541667
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,3,0.0665280024210612
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,3,0.05005866785844167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,7,0.06796800096829732
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,7,0.050186668833096824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,15,0.06754133105278015
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,15,0.05017599960168203
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,63,0.069882666071256
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,63,0.05091199775536855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,31,0.06889066596825917
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,31,0.05027199784914652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,127,0.1999733249346415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,127,0.052613332867622375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,255,0.38414398829142254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,255,0.19734932978947958
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,511,0.3817226489384969
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,511,0.7447893619537354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1,0.11202667156855266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1,0.09101866682370503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1023,1.6118933359781902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1023,0.787722667058309
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,3,0.1141973336537679
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,3,0.09117866555849712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,7,0.11582932869593303
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,7,0.09073066711425781
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,15,0.11570666233698527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,15,0.09121599793434143
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,63,0.20101332664489746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,63,0.09263466795285542
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,127,0.39134931564331055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,127,0.20401599009831747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,255,0.780789295832316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,31,0.11779733498891194
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,31,0.09158399701118469
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,255,0.38577600320180255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,511,1.6722399393717449
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,511,0.7956693172454834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,1,0.05042133231957754
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,1,0.06152533491452535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,3,0.062496001521746315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1023,1.5609280268351238
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1023,3.33189328511556
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,3,0.0505920002857844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,15,0.0633493314186732
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,15,0.0506933331489563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,7,0.06363200147946675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,7,0.05085866649945577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,31,0.06449066599210103
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,63,0.051498666405677795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,127,0.2015786568323771
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,127,0.05385600030422211
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,31,0.05073066552480062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,63,0.06548800071080525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,255,0.20030933618545532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,255,0.3858559926350911
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,511,0.38552534580230713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,1,0.12029866377512614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,511,0.7709866364796957
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,1,0.09047466516494751
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,3,0.09046933054924011
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,3,0.1218773325284322
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,7,0.12405332922935486
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,7,0.09066667159398396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,15,0.12362666924794515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,15,0.09062400460243225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,31,0.09101866682370503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,31,0.12777599692344666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,63,0.206821342309316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,63,0.09178133805592854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,127,0.3962613344192505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,127,0.20745599269866943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,255,0.7780266602834066
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,1,0.21876267592112222
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,1,0.17525867621103922
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,255,0.3925333420435588
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,511,1.6268320083618164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,511,0.7970933119455973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,3,0.22267733017603555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,3,0.17543999354044595
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,7,0.17562667528788248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,15,0.22828267018000284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,7,0.2264159917831421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,15,0.17571733395258585
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,63,0.40136531988779706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,63,0.21795733769734701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,31,0.2520479957262675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,31,0.17639466126759848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1,0.008896000062425932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1,0.011125333607196808
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,127,0.7747413317362467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,3,0.00897066667675972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,3,0.011039999624093374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,7,0.009018666421373686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,15,0.008869333192706108
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,15,0.011087999989589056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,31,0.00903466654320558
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,31,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,63,0.009056000038981438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,63,0.011253333340088526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,127,0.40556267897288006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,127,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,127,0.009194666519761086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,255,0.01250133290886879
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,255,0.01192533348997434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,511,0.013754667093356451
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1023,0.014074667046467463
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1023,0.016528000434239704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,2047,0.0198186660806338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,2047,0.02250666668017705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,4095,0.024682665864626568
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,4095,0.034474665919939675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,8191,0.03453333427508672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,7,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,8191,0.05842666824658712
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,16383,0.19358932971954346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,16383,0.10601066549619038
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,32767,0.3742773135503133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,32767,0.2131999929745992
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,511,0.012789333860079447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,65535,0.7317919731140137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,65535,0.41594131787618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,131071,1.6158347129821777
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,131071,0.8137866655985514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1,0.01109333336353302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,3,0.012330666184425354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,7,0.012351999680201212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,7,0.01110400011142095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,15,0.01239466667175293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,15,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,31,0.012378666549921036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,31,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,63,0.011221333096424738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,63,0.012639999389648438
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,127,0.012655999511480331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,127,0.011493333925803503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,255,0.012144000579913458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,255,0.015354666858911514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,511,0.01586666703224182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,255,1.6793333689371746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,511,0.013541333377361298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1023,0.021498667697111767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1023,0.01685333376129468
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,2047,0.026538667579491932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,2047,0.022624000906944275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,4095,0.040421334405740104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,8191,0.05858666698137919
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,8191,0.19376534223556519
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,16383,0.19128000736236572
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,16383,0.37491734822591144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,32767,0.7308106422424316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,32767,0.37142399946848553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,65535,0.7672746976216634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,65535,1.5311840375264485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1,0.00795199970404307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,255,0.762885332107544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1,0.011450666934251785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,3,0.007925333455204964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,7,0.007893333211541176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,3,0.011306667079528173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,131071,1.6076159477233887
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,15,0.008037333066264788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,15,0.011306667079528173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,131071,3.2222560246785483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,4095,0.03482666611671448
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,31,0.011178666104873022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,63,0.008298666526873907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,127,0.008629333227872849
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,31,0.008176000167926153
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,63,0.011274666835864386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,127,0.011658667276302973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,255,0.012554666648308435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,255,0.011050666371981302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,511,0.015674666812022526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1023,0.025034666061401367
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,511,0.014106666048367819
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,2047,0.0439573327700297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1023,0.017258666455745697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,4095,0.0359253336985906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,2047,0.023578666150569916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,4095,0.18920532862345377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,8191,0.3661280075709025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,8191,0.1913706660270691
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,16383,0.36720534165700275
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,16383,0.7241653601328532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,32767,0.7938133080800375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,32767,1.6304960250854492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,7,0.011424000064531961
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1,0.008682666967312494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,65535,1.5727626482645671
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1,0.011253333340088526
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,3,0.00879466657837232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,65535,3.24016539255778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,7,0.009029333169261614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,3,0.011087999989589056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,31,0.009008000294367472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,31,0.011130666981140772
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,15,0.008986666798591614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,63,0.00898133342464765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,63,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,131071,3.128239949544271
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,255,0.01258133351802826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,127,0.009381333366036415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,255,0.01293333371480306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,127,0.011370666325092316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,511,0.015765332927306492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1023,0.021781332790851593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,131071,6.492437362670898
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1023,0.018778666853904724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,2047,0.03366400053103765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,2047,0.02364266663789749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,4095,0.03745600084463755
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,4095,0.0576853354771932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,511,0.013408000270525614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,8191,0.1919893423716227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,8191,0.10532266894976298
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,511,1.5797173182169597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,511,3.3441174825032554
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,16383,0.2124320069948832
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,32767,0.4131893316904704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,16383,0.3729706605275472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,32767,0.831274668375651
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1,0.007807999849319458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,65535,1.6200052897135417
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,3,0.008266666904091835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,3,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,7,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,15,0.007957333077987036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,7,0.008031999692320824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,15,0.011306667079528173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,31,0.007877333089709282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,31,0.011173332730929056
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,131071,1.664181391398112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,63,0.008165333420038223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,63,0.01138666644692421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,127,0.008485333373149237
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,255,0.01313599944114685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,255,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,511,0.015423999478419622
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,131071,3.2311201095581055
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,511,0.016373333831628162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,127,0.011498666057984034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1023,0.022106667359670002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1023,0.025029333929220837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,4095,0.20680000384648642
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,2047,0.0342399999499321
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,4095,0.058133333921432495
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,8191,0.19057067235310873
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,8191,0.36047999064127606
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,16383,0.37459735075632733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,2047,0.04407466451327006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,32767,0.8143200079600016
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,16383,0.7188693682352701
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1,0.00850133349498113
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,65535,1.6061760584513347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1,0.010650667051474253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,3,0.010778666784365972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,7,0.008618666479984919
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,3,0.00867733359336853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,65535,3.189626693725586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,7,0.010661333799362183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,15,0.008709333216150602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,31,0.008746666833758354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,31,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,63,0.008559999987483025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,63,0.010672000547250112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,127,0.008842666943868002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,32767,1.5950506528218586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,127,0.010938666760921478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,255,0.0120319997270902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,15,0.01071999967098236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,131071,3.201274553934733
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,511,0.012319999436537424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,511,0.01166933278242747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1023,0.012917333592971167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,2047,0.011760000139474869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,2047,0.013973332941532135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,4095,0.015861333658297855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,8191,0.020479999482631683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,4095,0.012069333344697952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,16383,0.029152000943819683
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,8191,0.013493333011865616
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,16383,0.01658133293191592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,32767,0.03175999969244003
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,32767,0.02276800076166789
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,255,0.011711999773979187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,65535,0.03590933233499527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,131071,0.05565866827964783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,65535,0.033376000821590424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,131071,0.04655999938646952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1023,0.011578666667143503
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1,0.007525333513816197
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1,0.010773333410422007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,3,0.010527999450763067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,3,0.007578666632374127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,15,0.007696000238259633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,7,0.010757333288590113
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,7,0.007696000238259633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,31,0.007696000238259633
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,15,0.010559999694426855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,31,0.010784000158309937
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,63,0.007690666864315669
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,127,0.008031999692320824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,127,0.010938666760921478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,63,0.010661333799362183
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,255,0.010384000216921171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,131071,5.875722885131836
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,255,0.011477333803971609
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,511,0.01498666654030482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,511,0.01173866664369901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1023,0.01198400060335795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1023,0.024175999065240223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,2047,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,4095,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,4095,0.08018666505813599
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,8191,0.15476266543070474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,8191,0.01392000044385592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,16383,0.016832000265518825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,16383,0.30426667133967084
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,32767,0.02332266668478648
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,65535,0.03357866654793421
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,32767,0.6027520100275675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,65535,1.1994933287302654
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,131071,0.05602133274078369
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1,0.0084906667470932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1,0.010634666929642359
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,3,0.00860799973209699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,3,0.010751999914646149
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,7,0.00860799973209699
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,131071,2.429488023122152
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,15,0.01081066702802976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,15,0.008698666468262672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,31,0.008656000097592672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,31,0.011045332998037338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,127,0.009039999917149544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,63,0.010757333288590113
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,63,0.008736000085870424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,127,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,2047,0.04307200014591217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,255,0.011610666910807291
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,255,0.012170666207869848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,511,0.012346666306257248
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,511,0.011994666109482447
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1023,0.013034666577974955
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,2047,0.014202666779359182
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1023,0.011786667009194693
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,2047,0.012037333101034164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,4095,0.016496000190575916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,65535,0.8142240047454834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,8191,0.02107733239730199
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,8191,0.01658133293191592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,4095,0.01369599997997284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,16383,0.022426667312781017
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,16383,0.032885332902272545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,32767,0.03800000001986822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,65535,0.04855999847253164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,65535,0.05691199998060862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,131071,0.10171733299891154
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,32767,0.03455466777086258
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1,0.007514666765928268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,3,0.010847999403874079
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,3,0.007589333380262057
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,131071,0.22604799270629883
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,7,0.007786666974425316
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,15,0.007823999971151352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,15,0.01071999967098236
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,7,0.010879999647537867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,31,0.010816000401973724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,31,0.007770666852593422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,127,0.008058666562040647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,127,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,63,0.010773333410422007
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,255,0.01181866725285848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,63,0.007957333077987036
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,255,0.010384000216921171
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,511,0.014997333288192749
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,511,0.011727999895811081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1023,0.024282666544119518
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1023,0.011813333878914515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,2047,0.04299733539422353
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,4095,0.08031466603279114
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,4095,0.01369599997997284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,8191,0.15501333276430765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,2047,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,8191,0.01687466725707054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,16383,0.30431467294692993
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,16383,0.022837333381175995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,32767,0.03490666548411051
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,32767,0.6033493280410767
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,65535,0.057631999254226685
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,131071,0.19197332859039307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,65535,1.2187146345774333
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1,0.008789333204428354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1,0.011285333583752314
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,3,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,3,0.009093333035707474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,7,0.009216000015536943
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,7,0.011152000476916632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,15,0.009066666786869368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,131071,2.4308160146077475
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,15,0.011359999577204386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,31,0.00903466654320558
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,31,0.011333333949247995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,63,0.009258666386206945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,127,0.011733333269755045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,127,0.009472000102202097
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,63,0.011434666812419891
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,255,0.013194666554530462
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,255,0.013264000415802002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,511,0.016336000214020412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,511,0.018031999468803406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1023,0.023018665611743927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1023,0.0222080002228419
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,2047,0.034245334565639496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,2047,0.03316800047953924
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,8191,0.37249600887298584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,8191,0.19296000401178995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,4095,0.05819199979305267
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,4095,0.212719996770223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,16383,0.7280159791310629
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,16383,0.3696586688359578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1,0.011429333438475927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1,0.007946666950980822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,32767,0.7803626855214437
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,3,0.008373333141207695
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,3,0.011509332805871964
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,32767,1.444170633951823
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,7,0.008207999790708223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,15,0.011343999455372492
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,15,0.008421333506703377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,31,0.011503999431928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,7,0.011514666179815928
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,63,0.008357333640257517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,63,0.01179733375708262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,65535,3.2333545684814453
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,127,0.00891733355820179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,31,0.008346666892369589
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,127,0.012144000579913458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,255,0.01139733319481214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,255,0.013946666071812311
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,65535,1.444602648417155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,511,0.016965333372354507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1023,0.026608000199000042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1023,0.022991999983787537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,4095,0.19085333744684854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,2047,0.1854026714960734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,2047,0.03568533311287562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,511,0.01655999943614006
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,4095,0.40562132994333905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,8191,0.718010663986206
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,16383,0.7920800050099691
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1,0.008746666833758354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,8191,0.3675466775894165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,3,0.008789333204428354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,3,0.010672000547250112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1,0.01091733326514562
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,7,0.008842666943868002
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,7,0.010832000523805618
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,15,0.008933333059151968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,15,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,31,0.008832000195980072
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,63,0.008933333059151968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,31,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,63,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,127,0.009306666751702627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,127,0.011120000233252844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,255,0.011909333368142446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,255,0.01228800043463707
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,511,0.012762666990359625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,511,0.011952000359694162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1023,0.013295999417702356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,16383,1.48744535446167
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1023,0.01202133297920227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,2047,0.014607999473810196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,2047,0.013376000026861826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,4095,0.016480000068744022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,8191,0.02629866699377696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,8191,0.022592000663280487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,16383,0.031130666534105938
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,32767,0.058287998040517174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,32767,0.040965333580970764
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,16383,0.03470933437347412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,65535,0.10596799850463867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,131071,0.3843839963277181
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,65535,0.2196213404337565
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,131071,0.21396267414093018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1,0.008698666468262672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1,0.010922666639089584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,3,0.008122666428486506
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,4095,0.017184000462293625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,3,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,7,0.007871999715765318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,7,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,15,0.008336000144481659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,31,0.00784533346692721
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,31,0.01099733387430509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,15,0.011098666737476984
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,63,0.00797333319981893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,63,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,127,0.008432000254591307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,255,0.011989332735538483
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,255,0.010570666442314783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,127,0.01128000020980835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,511,0.01227733368674914
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,511,0.015189333508412043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,2047,0.043231998880704246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,2047,0.013770667215188345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1023,0.01231466606259346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1023,0.024656000236670177
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,4095,0.080485333998998
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,4095,0.016773333152135212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,8191,0.0229066660006841
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,8191,0.1553813318411509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,16383,0.3043893376986186
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,16383,0.03498133271932602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,32767,0.05867200096448263
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,65535,0.1913386583328247
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,65535,1.2202400366465251
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,32767,0.6137066682179769
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,131071,2.4314400355021157
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,131071,0.378762682278951
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1,0.009381333366036415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,3,0.009541333342591921
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1,0.01166933278242747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,3,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,7,0.009423999736706415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,7,0.011440000186363855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,15,0.009461333354314169
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,15,0.011802667131026586
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,31,0.009514666472872099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,31,0.011589333415031433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,32767,1.4328640302022297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,32767,2.8453493118286133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,63,0.009568000212311745
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,63,0.011999999483426413
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,127,0.01240533341964086
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,127,0.010288000106811523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,65535,3.117151896158854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,511,0.017184000462293625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,511,0.0229120006163915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,65535,6.395402908325195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,255,0.01842133328318596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1023,0.03297066688537598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1023,0.02327466756105423
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,255,0.013978666315476099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,2047,0.03597866743803024
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,8191,0.369706670443217
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,8191,0.7287413279215494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,4095,0.37215999762217206
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,16383,0.725050687789917
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,16383,1.4427092870076497
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1,0.012122667084137598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,3,0.011861333002646765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,2047,0.2124533255894979
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,7,0.012170666207869848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,7,0.013514666507641474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,3,0.013797332843144735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1,0.013546666751305262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,4095,0.20898133516311646
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,32767,1.5729279518127441
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,31,0.012117333710193634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,15,0.013754667093356451
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,31,0.013898666948080063
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,63,0.012298667182525
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,32767,2.8747145334879556
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,15,0.011877333124478659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,127,0.014864000181357065
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,255,0.017781333376963932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,255,0.018592000007629395
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,127,0.013338666409254074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,511,0.027850667635599773
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,63,0.01423466702302297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1023,0.18616533279418945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,4095,0.7192906538645426
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1023,0.03299733251333237
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,511,0.022997332115968067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,4095,0.3700480063756307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,8191,1.4612639745076497
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,2047,0.38811198870340985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,2047,0.19370667139689127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1,0.01302933320403099
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,8191,0.725104014078776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,16383,3.2080745697021484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,3,0.013936000565687815
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1,0.01393066719174385
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,16383,1.4298720359802246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,3,0.013274667163689932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,7,0.013306666165590286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,32767,3.0824639002482095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,15,0.013306666165590286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,15,0.013855999956528345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,7,0.013690666606028875
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,31,0.014117332796255747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,63,0.013818666338920593
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,31,0.013429333766301474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,127,0.014736000448465347
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,32767,6.17193603515625
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,127,0.015317333241303762
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,255,0.019946667055288952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,255,0.018165333817402523
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,511,0.029253333806991577
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,511,0.023141334454218548
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,63,0.014309333016475042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1023,0.18946133057276407
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,2047,0.36900798479715985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,2047,0.1947093407313029
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,8191,1.4392107327779133
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1,0.019061333189407986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,8191,0.7620853583017985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1023,0.03365333378314972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1,0.020026666422684986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,3,0.019866666446129482
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,4095,0.39472532272338867
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,3,0.019205333044131596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,4095,0.8122399648030599
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,7,0.019493332753578823
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,7,0.02011200040578842
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,15,0.01941866676012675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,15,0.020047999918460846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,31,0.01971199984351794
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,63,0.020261333634455998
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,127,0.021594665944576263
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,127,0.022240000466505688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,255,0.026613332331180573
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,255,0.03181333343187968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,511,0.19973333676656088
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,31,0.020047999918460846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,63,0.020303999384244282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,511,0.036346666514873505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1023,0.1964319944381714
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1023,0.364896019299825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,4095,1.5829119682312012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,4095,0.7316640218098959
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,2047,0.7187466621398926
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,2047,0.3988800048828125
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,8191,1.555023988087972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1,0.020693333198626835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,8191,2.8420159022013345
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1,0.02063999945918719
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,3,0.02057066683967908
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,3,0.020853333175182343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,15,0.020453333854675293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,63,0.02086399992307027
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,31,0.020634666085243225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,63,0.02182399978240331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,15,0.021018666525681812
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,7,0.02048533285657565
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,31,0.02146666745344798
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,7,0.021216000119845074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,127,0.023541333774725597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,127,0.021989333132902782
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,255,0.03494933247566223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,255,0.02762666592995326
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,511,0.19006399313608804
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,511,0.03689600030581156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1023,0.1962453325589498
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1023,0.3704746564229329
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1,0.03336533407370249
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1,0.031925333042939506
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,2047,0.813477357228597
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,2047,0.3751946687698364
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,3,0.033770665526390076
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,3,0.03206400076548258
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,7,0.034416000048319496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,7,0.03180266668399175
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,15,0.034517332911491394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,31,0.03498133271932602
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,31,0.03216533362865448
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,15,0.03205333401759466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,4095,1.6149013837178547
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,4095,0.7871306737263998
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,63,0.03570666660865148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,63,0.03215466688076655
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,127,0.037765334049860634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,127,0.033200000723203026
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,255,0.18756266434987387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,255,0.043807998299598694
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1023,0.3777066469192505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1023,0.7479360103607178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,4095,1.4395306905110676
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,4095,2.8571627934773765
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,511,0.36910398801167804
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,511,0.19784533977508545
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1,0.008757333581646284
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,3,0.008816000074148178
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,3,0.010960000256697336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,15,0.008901333436369896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,31,0.008943999807039896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,31,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,15,0.010853332777818045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,63,0.008938666433095932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,63,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,127,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,255,0.012319999436537424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,127,0.009328000247478485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,511,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,511,0.011866666376590729
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1023,0.01358933374285698
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1023,0.013376000026861826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,2047,0.015072000523408255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,2047,0.01637866720557213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,4095,0.021925332645575207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,4095,0.02232533444960912
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,8191,0.026799999177455902
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,8191,0.034154665966828666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,7,0.009002666920423508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,16383,0.036517334481080375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,16383,0.058176000912984215
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,7,0.01102399950226148
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,32767,0.21570666631062826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,32767,0.10612799723943074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,65535,0.21354132890701294
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,65535,0.3773226737976074
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1,0.008634666601816813
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1,0.010922666639089584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,3,0.00784533346692721
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,131071,0.8167253335316976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,131071,0.425653338432312
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,7,0.007802666475375493
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,7,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,15,0.007936000203092894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,31,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,15,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,63,0.0081386665503184
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,63,0.01110400011142095
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,31,0.007903999959429106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,127,0.008112000301480293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,255,0.010527999450763067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,127,0.011194666226704916
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,255,0.01202133297920227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,511,0.015077333897352219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,511,0.012144000579913458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1023,0.02455466737349828
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1023,0.013663999736309052
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,2047,0.04320533573627472
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,2047,0.016528000434239704
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,4095,0.08054399987061818
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,4095,0.022821334501107533
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,8191,0.15507733821868896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,8191,0.034901333351929985
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,16383,0.05855466425418854
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,16383,0.3109653393427531
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,32767,0.19060800472895303
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,32767,0.6167466640472412
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,65535,0.3689226706822713
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,65535,1.2263360023498535
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,131071,0.8154666423797607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,2047,0.7329706350962321
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,2047,1.6024959882100422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,131071,2.445631980895996
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1,0.035216001172860466
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1,0.029637334247430164
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,3,0.035946667194366455
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,3,0.029653333127498627
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,15,0.036159999668598175
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,15,0.029696000119050343
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,7,0.036159999668598175
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,7,0.029765332738558452
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,31,0.036789332826932274
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,31,0.029781334102153778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,63,0.03756800045569738
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,63,0.029829333225886028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,127,0.039290666580200195
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,127,0.03081600119670232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,255,0.19244800011316934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,255,0.042165334026018776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,511,0.37428800264994305
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,511,0.19483733177185059
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1023,0.7334293524424235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1023,0.37435734272003174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1,0.059248000383377075
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1,0.04987200101216634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,2047,1.6191360155741374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,3,0.060506666700045265
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,3,0.0498933345079422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,2047,0.7851733366648356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,15,0.061205332477887474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,7,0.0618399977684021
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,7,0.04986133178075155
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,31,0.050101334849993386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,63,0.06386666496594746
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,127,0.18977065881093344
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,31,0.062181333700815834
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,127,0.05303466816743215
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,15,0.05011733373006185
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,255,0.40987733999888104
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,63,0.05057600140571594
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,255,0.196560005346934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,511,0.7679146925608317
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,511,0.37882133324941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1,0.06116800010204315
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1023,0.7318933010101318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1,0.05035200218359629
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1023,1.5992693901062012
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,3,0.06242666641871134
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,2047,1.5492159525553386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,2047,3.197882652282715
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,3,0.050240000089009605
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,15,0.06318933268388112
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,7,0.05022933085759481
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,31,0.06451733410358429
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,7,0.06332799792289734
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,15,0.05026666820049286
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,63,0.05096533397833506
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,127,0.19594667355219522
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,127,0.05387733379999796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,31,0.05055999755859375
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,255,0.3787253300348918
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,63,0.06586666901906331
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,255,0.19769599040349325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,511,0.7372159957885742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,511,0.4065759976704915
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1,0.11196800072987874
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1,0.09000533819198608
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,3,0.11442133784294128
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1023,1.6183253924051921
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1023,0.7879947026570638
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,3,0.08974400162696838
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,7,0.1162453293800354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,15,0.11563199758529663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,15,0.09044266740481059
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,7,0.09019200007120769
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,31,0.11831999818483989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,63,0.09145599603652954
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,127,0.37748801708221436
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,31,0.09074133634567261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,63,0.19358932971954346
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1,0.00895999992887179
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,127,0.20404799779256186
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,3,0.00897066667675972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,7,0.008943999807039896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,7,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,15,0.008901333436369896
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,15,0.011071999867757162
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,31,0.009343999748428663
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,3,0.011141333729028702
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,63,0.009152000149091085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,63,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,127,0.009301333377758661
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,127,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,255,0.012389333297808966
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,255,0.011946666985750198
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,511,0.012901333471139273
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,511,0.013429333766301474
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,255,0.8135039806365967
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1023,0.014021333307027817
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1023,0.016506666938463848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,2047,0.019648000597953796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,255,0.4138186772664388
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,2047,0.0223786657055219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,4095,0.03431999931732813
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,8191,0.03443733354409536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,16383,0.10584533214569092
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,16383,0.19284266233444214
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,32767,0.37350932757059735
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,32767,0.21361599365870157
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,65535,0.41471465428670246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,65535,0.7738880316416422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,131071,1.6305599212646484
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,131071,0.8291520277659098
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,4095,0.025199999411900837
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1,0.007760000104705493
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1,0.010938666760921478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,3,0.007871999715765318
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,3,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,8191,0.058464000622431435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,7,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,15,0.007850666840871176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,7,0.008047999814152718
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,15,0.011215999722480774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,31,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,31,0.007802666475375493
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,63,0.008090666805704435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,63,0.011183999478816986
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,127,0.008357333640257517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,127,0.01126933346192042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,255,0.010656000425418219
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,255,0.012181332955757776
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,511,0.015376000354687372
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,511,0.01358933374285698
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1023,0.024714666108290356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1023,0.016821333517630894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,2047,0.043354665239652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,4095,0.0807360013326009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,2047,0.022837333381175995
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,511,1.4414879480997722
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,4095,0.0349386657277743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,8191,0.05866133173306783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,8191,0.18522133429845175
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,511,0.7407413323720297
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,16383,0.1917440096537272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,16383,0.35990933577219647
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,32767,0.37043734391530353
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,65535,0.803429365158081
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1,0.007882666463653246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,65535,1.5866400400797527
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,3,0.007903999959429106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1,0.010933333386977514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,3,0.011045332998037338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,7,0.007893333211541176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,7,0.011034666250149408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,15,0.007823999971151352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,15,0.011055999745925268
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,31,0.00789866658548514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,131071,3.1972427368164062
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,31,0.01099733387430509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,63,0.008037333066264788
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,127,0.011301333705584208
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,127,0.008357333640257517
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,63,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,255,0.012869333227475485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,255,0.019648000597953796
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,511,0.01581866666674614
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1023,0.024570666253566742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,511,0.015381333728631338
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1023,0.021727999051411945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,2047,0.04334400097529093
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,2047,0.03365866591533025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,4095,0.08060800035794576
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,4095,0.05760000149408976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,8191,0.1873706579208374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,32767,0.7216266791025797
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,8191,0.10541333754857381
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,16383,0.21405333280563354
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,16383,0.36708799997965497
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,32767,0.7295146783192953
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,32767,0.4132159948348999
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1,1.0359893639882405
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,65535,1.4440746307373047
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,131071,1.4309226671854656
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1,0.010853332777818045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,7,0.008154666672150293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,3,0.008565333361426989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,7,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,15,0.00790933333337307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,3,0.010922666639089584
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,15,0.010863999525705973
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,31,0.007733333234985669
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,31,0.010757333288590113
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,63,0.010869332899649939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,63,0.008112000301480293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,127,0.008026666939258575
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,127,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,255,0.010373333469033241
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,255,0.011674666156371435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,511,0.011877333124478659
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,511,0.014949332922697067
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1023,0.024314666787783306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1023,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,2047,0.042992000778516136
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1023,3.19374942779541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,2047,0.011909333368142446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,4095,0.012191999703645706
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,4095,0.08040000001589458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,8191,0.15516266226768494
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,8191,0.013674666484196981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,16383,0.30435200532277423
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,16383,0.016741332908471424
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,65535,0.033226666351159416
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,32767,0.6029920180638632
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,65535,1.2003573576609294
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,131071,0.05583466589450836
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1,0.010837333897749582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1,0.00892800030608972
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,3,0.007840000092983246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,131071,2.3929707209269204
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,3,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,7,0.011002667248249054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,15,0.009018666421373686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,7,0.009381333366036415
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,31,0.009018666421373686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,63,0.008298666526873907
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,63,0.01109333336353302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1023,1.5583573977152507
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,127,0.008170666793982187
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,255,0.010431999961535135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,255,0.011887999872366587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,511,0.015029333531856537
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,511,0.011701333026091257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1023,0.01181866725285848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,2047,0.04312000175317129
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,2047,0.012149333953857422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,4095,0.0803306649128596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,4095,0.01360000049074491
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,8191,0.15520000457763672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,8191,0.016501333564519882
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,65535,0.8276960055033366
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,16383,0.30456533034642536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,16383,0.022592000663280487
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,32767,0.6034239927927653
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,65535,0.05695466697216034
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,32767,0.03482133398453394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,65535,1.2004586855570476
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,131071,0.10172266761461894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,131071,1.6619146664937336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1,0.007840000092983246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,131071,2.4321813583374023
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,3,0.00797333319981893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,131071,3.230112075805664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,7,0.00797333319981893
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,15,0.007978666573762894
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,7,0.011349332829316458
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,15,0.011391999820868174
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,31,0.008000000069538752
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,3,0.01146666705608368
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,31,0.011242666592200598
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,63,0.008367999767263731
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,63,0.011296000331640244
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,127,0.008517333616813024
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,127,0.011754666765530905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,255,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,255,0.013450667262077332
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,511,0.016106666376193363
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,511,0.015599999576807022
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,32767,0.022858666876951855
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1023,0.022170667846997578
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1023,0.025253333151340485
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,2047,0.03421866645415624
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,2047,0.04399999976158142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,8191,0.364901343981425
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,8191,0.19339734315872192
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,4095,0.1874720056851705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,4095,0.058320000767707825
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,16383,0.7296746571858724
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,32767,0.8154079914093018
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,32767,1.5329492886861165
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,16383,0.36957331498463947
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1,0.007797333101431529
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,3,0.007882666463653246
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,7,0.007770666852593422
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,15,0.007850666840871176
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,31,0.007776000226537387
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,31,0.011029332876205444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,15,0.011050666371981302
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,63,0.011061333119869232
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,63,0.007887999837597212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,127,0.008074666683872541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,127,0.011274666835864386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,255,0.01033599985142549
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,255,0.012015999605258306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,65535,1.4441812833150227
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,511,0.013877333452304205
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,511,0.015178666760524115
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1023,0.024405332903067272
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,2047,0.04331199824810028
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,65535,2.859365463256836
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1023,0.013354666531085968
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,4095,0.0804906686147054
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,2047,0.013669333110253016
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,8191,0.15595199664433798
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,4095,0.016645333419243496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1023,0.024341332415739696
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,8191,0.022341333329677582
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,16383,0.034527999659379326
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,32767,0.6032319863637289
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,65535,0.10604799787203471
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,16383,0.30452799797058105
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,32767,0.05849599838256836
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,131071,0.21798400084177652
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,65535,1.221951961517334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,131071,2.434384028116862
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1,0.008074666683872541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1,0.011605333536863327
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,131071,3.219914754231771
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,3,0.008245333408315977
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,7,0.008250666782259941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,3,0.011535999675591787
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,15,0.008223999912540117
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,7,0.011616000284751257
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,15,0.011589333415031433
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,31,0.011695999652147293
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,63,0.008687999720374743
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,63,0.011706666400035223
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,131071,6.399728139241536
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,127,0.009045333291093508
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,127,0.012362666428089142
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,255,0.011450666934251785
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,255,0.014015999933083853
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,511,0.01640533283352852
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,511,0.016970666746298473
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,31,0.008394666636983553
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1023,0.023013333479563396
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,2047,0.20829333861668906
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,2047,0.035589332381884255
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,8191,0.7246346473693848
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,8191,0.40330668290456134
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,4095,0.19288533926010132
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,4095,0.36540265878041583
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,16383,1.4339466094970703
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1023,0.026954665780067444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1,0.013568000247081121
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,16383,0.7252853711446127
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,3,0.01201066623131434
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,32767,1.5741920471191406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1,0.011834666132926941
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,3,0.013525333255529404
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,32767,2.8619041442871094
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,15,0.012069333344697952
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,7,0.012186666329701742
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,15,0.0136266661187013
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,7,0.013717333475748697
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,65535,6.432320276896159
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,31,0.013663999736309052
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,63,0.012608000387748083
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,31,0.012560000022252401
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,65535,2.8644212086995444
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,63,0.014111999422311783
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,127,0.013653332988421122
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,255,0.018559999763965607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,255,0.017994667092959087
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,511,0.02863466739654541
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1023,0.1871946652730306
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1023,0.03326933334271113
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,127,0.015050667027632395
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,2047,0.36657599608103436
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,2047,0.19381332397460938
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,8191,1.4383519490559895
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,511,0.023045333723227184
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,8191,0.7278453509012858
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,4095,0.3708053429921468
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,4095,0.8106133143107096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1,0.020047999918460846
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,16383,1.4363519350687664
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1,0.019098666807015736
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,16383,3.2097546259562173
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,3,0.019487999379634857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,3,0.020074666788180668
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,15,0.019519999623298645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,15,0.020143999407688778
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,31,0.0199946661790212
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,63,0.020314666132132213
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,63,0.020421333611011505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,31,0.02024000013868014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,7,0.019930666933457058
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,7,0.020058666666348774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,127,0.02197333425283432
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,127,0.02187199890613556
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,255,0.03209066639343897
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,255,0.027087998886903126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,511,0.20086934169133505
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,511,0.036570665736993156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1023,0.40833067893981934
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1023,0.19489065806070963
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1,0.00772266648709774
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1,0.011039999624093374
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,3,0.007738666608929634
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,7,0.007749333356817563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,3,0.011066666493813196
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,7,0.010768000036478043
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,2047,0.7669226328531901
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,15,0.008656000097592672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,15,0.010858666151762009
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,31,0.010821333775917688
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,63,0.007920000081261
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,63,0.011157333850860596
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,127,0.008133333176374435
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,127,0.011146667102972666
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,255,0.010431999961535135
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,8191,1.43941863377889
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,511,0.015034666905800501
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,511,0.01179733375708262
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,2047,0.37355732917785645
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1023,0.013418667018413544
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1023,0.02441066751877467
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,4095,0.08040533463160197
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,4095,0.02237333357334137
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,8191,2.860581398010254
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,8191,0.034245334565639496
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,8191,0.155157337586085
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,16383,0.05813866853713989
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,32767,0.6144959926605225
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,32767,0.10583999752998352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,65535,0.21364800135294595
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,2047,0.04318933188915253
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,65535,1.2215733528137207
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,16383,0.30473599831263226
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,131071,0.41492267449696857
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,131071,2.4382880528767905
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,31,0.00786666696270307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,2047,0.01658133293191592
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,4095,1.6127626101175945
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,4095,0.7863626480102539
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1,0.033557333052158356
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1,0.0296426663796107
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,7,0.034602666894594826
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,3,0.029450667401154835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,3,0.03417599946260452
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,7,0.029663999875386555
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,31,0.03526933242877325
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,63,0.035616000493367515
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,31,0.029605334003766377
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,63,0.029648000995318096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,127,0.037845333417256675
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,127,0.030784000953038532
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,15,0.03484266748030981
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,15,0.02961066613594691
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,255,0.18971200784047446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,255,0.04101333270470301
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,511,0.1976906657218933
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,511,0.4095573425292969
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1023,0.8134826819101969
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1023,0.3736533323923747
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,2047,1.6149226824442546
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,2047,0.783023993174235
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1,0.059477334221204124
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1,0.04985066751639048
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,4095,1.5507094065348308
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,3,0.06063466767470042
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,4095,3.226186752319336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,3,0.050101334849993386
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,7,0.06217599908510844
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,15,0.061386664708455406
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,15,0.05016533533732096
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,7,0.050154666105906166
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,63,0.06358933448791504
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,63,0.05093866586685181
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,127,0.19239999850591025
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,127,0.05324266850948334
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,255,0.19642666975657144
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,255,0.40793601671854657
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,31,0.062218666076660156
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1,0.007727999861041705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,31,0.05027733246485392
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1,0.010869332899649939
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,3,0.007727999861041705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,7,0.007877333089709282
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,3,0.011077333241701126
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,7,0.010874666273593903
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,15,0.007946666950980822
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,15,0.010938666760921478
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,31,0.007903999959429106
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,31,0.01097600037852923
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,63,0.008031999692320824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,63,0.010949333508809408
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,127,0.008261333530147871
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,127,0.011018666128317514
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,255,0.010597333312034607
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,255,0.011887999872366587
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,511,0.7703626950581869
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,511,0.01505600040157636
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,511,0.013386666774749756
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1023,0.02456533412138621
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1023,0.016437333077192307
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,511,0.37781866391499835
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,2047,0.02233600119749705
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,4095,0.08041066428025563
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,4095,0.03435733417669932
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,8191,0.05806399881839752
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,16383,0.31173866987228394
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,16383,0.10583999752998352
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,32767,0.6177706718444824
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,32767,0.21260799964269003
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,65535,1.2248000303904216
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,65535,0.4126880168914795
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,2047,0.043466667334238686
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,8191,0.1553813318411509
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,131071,0.8339946269989014
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,131071,2.4490720431009927
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1023,1.6120586395263672
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,2047,3.2186880111694336
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1023,0.7846826712290446
VLLM,0.12.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,2047,1.5501920382181804
