framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1,0.009759999811649323
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,15,0.00983466642598311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,31,0.012383999923865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,63,0.010133333504199982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,63,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,127,0.010015999898314476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,127,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,3,0.011039999624093374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,255,0.014570667097965876
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,511,0.016021333634853363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,511,0.0169813334941864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,7,0.00983466642598311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1023,0.023018665611743927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,2047,0.0277813325325648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,2047,0.03499733408292135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,4095,0.041450666884581246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,4095,0.05881066620349884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1,0.013424000392357508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,1,0.0124746672809124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,3,0.013562666873137156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,1,0.01240533341964086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,3,0.012367999802033106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,7,0.012517333030700684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,7,0.013807999591032663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,15,0.013637332866589228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,15,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,31,0.013701333353916803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,31,0.012282667060693106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,63,0.013807999591032663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,63,0.012469333906968435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,127,0.014159999787807465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,127,0.012752000242471695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,255,0.017637333522240322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,3,0.012437333663304647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,255,0.01431999976436297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,511,0.023178666830062866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,511,0.01739199956258138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1023,0.028309332827727
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,1023,0.0234400009115537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,255,0.014384000251690546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,2047,0.03536533315976461
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,4095,0.2026240030924479
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,2,128,1,float16,fp8,4095,0.059578667084376015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,31,0.009898666913310686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1,0.012645332763592402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,2047,0.03865066667397817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,7,0.012549333274364471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,3,0.013717333475748697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1,0.013663999736309052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,7,0.013610667238632837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,7,0.012554666648308435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,1023,0.023189333577950794
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,15,0.012863999853531519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,31,0.013744000345468521
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,15,0.013946666071812311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,31,0.012554666648308435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,63,0.013989333063364029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,127,0.014639999717473984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,127,0.013264000415802002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,255,0.014848000059525171
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,63,0.012837332983811697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,255,0.02236266682545344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,511,0.027322667340437572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,511,0.018005333840847015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1023,0.03705599904060364
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,2047,0.03693866729736328
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,1023,0.024336000283559162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,2047,0.22499734163284302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,4095,0.1950826644897461
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,4095,0.39614399274190265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,3,0.01730666682124138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,3,0.014677333335081736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1,0.01463466634353002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1,0.01700266698996226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,7,0.01747200017174085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,7,0.014746667196353277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,15,0.017407999684413273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,15,0.01488000030318896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,1,128,1,float16,fp8,15,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,31,0.017535999417304993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,31,0.014709333578745524
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,63,0.015066667149464289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,63,0.017887999614079792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,127,0.0161920003592968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,127,0.01929066702723503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1023,0.2019360065460205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,255,0.018805333723624546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,511,0.03349866718053818
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,511,0.024112001061439514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,255,0.024570666253566742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,1023,0.03442666679620743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,2047,0.38889066378275555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,1,0.010746666540702185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,2047,0.19716266791025797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,3,0.009141333401203156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,3,0.010911999891201654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,7,0.008901333436369896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,15,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,7,0.010757333288590113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,4095,0.7583253383636475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,15,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,31,0.009050666665037474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,31,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,8,128,1,float16,fp8,4095,0.3763999938964844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,63,0.009093333035707474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,255,0.013365333278973898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,127,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,127,0.009242666885256767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,63,0.010725333044926325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,511,0.013088000317414602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,255,0.01163200040658315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1023,0.013616000612576803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,1023,0.011706666400035223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,511,0.011424000064531961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,2047,0.014736000448465347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,4095,0.017082666357358296
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,2047,0.011936000237862269
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,4095,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,8191,0.021664001047611237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,16383,0.03163733333349228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,8191,0.013605333864688873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,1,128,1,float16,fp8,16383,0.016693333784739178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,7,0.008639999975760778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,1,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,3,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,7,0.01099733387430509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,3,0.008725333337982496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,15,0.00877333308259646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,31,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,15,0.010661333799362183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,31,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,63,0.008954666554927826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,63,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,255,0.012357333054145178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,127,0.009082666908701261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,127,0.010922666639089584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,511,0.012826666235923767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,255,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,511,0.011749333391586939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1023,0.013264000415802002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,2047,0.014565333724021912
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,1023,0.011706666400035223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,2047,0.012181332955757776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,4095,0.016890666137139004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,4095,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,8191,0.02178666740655899
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,16383,0.03555733213822047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,8191,0.013936000565687815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1,0.008634666601816813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,2,128,1,float16,fp8,16383,0.017071999609470367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,3,0.008629333227872849
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,7,0.00878399983048439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,3,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,15,0.008821333448092142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,7,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,31,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,15,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,63,0.00890666681031386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,127,0.009077333534757296
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,63,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,31,0.0107893335322539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,255,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,127,0.011039999624093374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,255,0.011829332758982977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,511,0.012714666624863943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,511,0.011978667229413986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,2047,0.014767999450365702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,1023,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,4095,0.017360000560681026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,2047,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1023,0.013290667285521826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,4095,0.013077333569526672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,8191,0.026421333352724712
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,16383,0.031386665999889374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,8191,0.014762666076421738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,4,128,1,float16,fp8,16383,0.017685333887736004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,7,0.008746666833758354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1,0.008613333106040955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,3,0.011402666568756104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,3,0.00874133345981439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,15,0.008826666822036108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,7,0.011274666835864386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,64,4,128,1,float16,fp8,3,0.012773333738247553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,31,0.008842666943868002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,15,0.010933333386977514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,63,0.008890666688481966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,31,0.011440000186363855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,63,0.011264000087976456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,255,0.012367999802033106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,127,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1023,0.013530666629473368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,511,0.013936000565687815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,127,0.009018666421373686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,511,0.012698666503032049
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,1023,0.014352000008026758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,255,0.01322666679819425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,4095,0.021967999637126923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,2047,0.014533333480358124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,2047,0.015103999525308609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,4095,0.015717333803574245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,8191,0.026906666656335194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,8191,0.0183146670460701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,16383,0.036602665980656944
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,64,8,128,1,float16,fp8,16383,0.023685333629449207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,1,0.010842667271693548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,3,0.008986666798591614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,7,0.008986666798591614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,7,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,3,0.010858666151762009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,15,0.009088000282645226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,31,0.009130666653315226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,63,0.009178666397929192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,31,0.011045332998037338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,255,0.012714666624863943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,127,0.011215999722480774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,63,0.010784000158309937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,127,0.009242666885256767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,511,0.013093333691358566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,511,0.011877333124478659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1023,0.013647999614477158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,1023,0.011978667229413986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,2047,0.014794666320085526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,2047,0.012181332955757776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,4095,0.01748266691962878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,8191,0.022954667607943218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,4095,0.013610667238632837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,8191,0.016719999412695568
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,1,128,1,float16,fp8,16383,0.022543999056021374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,16383,0.03721600025892258
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1,0.012170666207869848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,1,0.010965333630641302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,3,0.01089599976936976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,7,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,3,0.012378666549921036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,7,0.01081066702802976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,15,0.012245333443085352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,31,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,15,0.010816000401973724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,63,0.012357333054145178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,31,0.01089599976936976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,63,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,127,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,127,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,255,0.014943999548753103
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,511,0.015360000232855478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,255,0.01181866725285848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,511,0.011989332735538483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1023,0.016010666886965435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,1023,0.012005332857370377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,2047,0.01724799970785777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,4095,0.02014933278163274
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,2047,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,4095,0.01381333296497663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,8191,0.02974933385848999
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,8191,0.016805333395799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,16383,0.03474666674931844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,2,128,1,float16,fp8,16383,0.023082666099071503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1,0.012053333222866058
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,3,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,7,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,3,0.011077333241701126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,15,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,31,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,15,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,31,0.011029332876205444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,63,0.012389333297808966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,127,0.012639999389648438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,63,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,255,0.014890667051076889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,127,0.011183999478816986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,255,0.011946666985750198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,511,0.015322666615247726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1023,0.016021333634853363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,511,0.012272000312805176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,1023,0.012240000069141388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,2047,0.01758933315674464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,2047,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,4095,0.02470933397610982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,8191,0.029738667110602062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,4095,0.014357333381970724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,8191,0.017525333911180496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,16383,0.039887999494870506
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,4,128,1,float16,fp8,16383,0.023738667368888855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,3,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1,0.011242666592200598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,7,0.012309333930412928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1,0.011968000481526056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,15,0.012240000069141388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,7,0.011391999820868174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,3,0.011813333878914515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,15,0.011370666325092316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,31,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,31,0.011829332758982977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,63,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,63,0.011600000162919363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,127,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,255,0.01413333291808764
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,255,0.015018666783968607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,511,0.015295999745527903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,511,0.014298666268587112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,127,0.011920000116030375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1023,0.01659199967980385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,4095,0.02735466758410136
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,1023,0.01461333284775416
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,2047,0.015450666348139444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,2047,0.022181332111358643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,8191,0.03730666637420654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,8191,0.023557332654794056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,4095,0.018346666047970455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,16383,0.20296533902486166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,64,8,128,1,float16,fp8,16383,0.0335359995563825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1,0.009733333562811216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,3,0.013429333766301474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,1,0.013370666652917862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,3,0.009749333063761393
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,7,0.013167999684810638
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,7,0.009808000177145004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,15,0.009786666681369146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,15,0.013381333400805792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,31,0.009919999788204828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,31,0.013274667163689932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,63,0.010117333382368088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,63,0.013327999661366144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,127,0.010442666709423065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,127,0.013728000223636627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,255,0.01695999999841054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,255,0.01515199989080429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,511,0.018298666924238205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,511,0.023183998962243397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1023,0.02808533360560735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,1023,0.024362665911515553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1,0.01451733335852623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,2047,0.038405333956082664
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,1,0.013317332913478216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,1,128,1,float16,fp8,2047,0.036229332288106285
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,3,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,3,0.014607999473810196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,7,0.013461332768201828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,15,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,7,0.01462399959564209
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,31,0.013386666774749756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,31,0.014709333578745524
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,15,0.014592000593741735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,63,0.013568000247081121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,63,0.015061333775520325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,127,0.014138666292031607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,127,0.015781333049138386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,255,0.02492800106604894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,255,0.015722667177518208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,511,0.018826667219400406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,511,0.029824001093705494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1023,0.039594667653242745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,1023,0.02516799916823705
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1,0.015477333217859268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1,0.017952000101407368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,3,0.018085333208243053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,3,0.01540800059835116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,2047,0.20724799235661825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,15,0.018170667191346485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,2,128,1,float16,fp8,2047,0.03761066744724909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,7,0.018309333672126133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,7,0.015696000307798386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,15,0.01568000018596649
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,31,0.01844266677896182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,31,0.015557333827018738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,63,0.016037333756685257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,127,0.017231999586025875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,255,0.01977066695690155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,63,0.018672000616788864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,511,0.024986666937669117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,511,0.0346666673819224
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,255,0.02492266645034154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,127,0.01988799994190534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,2047,0.20085867245992026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,2047,0.3943093220392863
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,4,128,1,float16,fp8,1023,0.03517866631348928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1023,0.20468266805013022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,3,0.021583999196688335
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1,0.024911999702453613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1,0.021482666333516438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,3,0.025381334125995636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,7,0.021642667551835377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,7,0.02587733417749405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,31,0.021536000072956085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,63,0.026352000733216602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,31,0.025573333104451496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,63,0.02197866638501485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,127,0.02805333336194356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,127,0.023434666295846302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,15,0.02145066608985265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,15,0.025493333737055462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,255,0.028549333413441975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,255,0.03787733366092046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,1023,0.19937600692113241
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,2047,0.764410654703776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1023,0.39581334590911865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1,0.008842666943868002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,1,0.010954666882753372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,511,0.03847466657559077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,7,0.009029333169261614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,3,0.009082666908701261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,3,0.011114666859308878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,7,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,511,0.2063466707865397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,15,0.011130666981140772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,31,0.009114666531483332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,64,8,128,1,float16,fp8,2047,0.3819733460744222
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,63,0.009194666519761086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,31,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,127,0.00949866697192192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,127,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,15,0.009066666786869368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,255,0.012874666601419449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,255,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,511,0.012063999970753988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,511,0.013327999661366144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1023,0.013872000078360239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,1023,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,63,0.01118933285276095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,2047,0.013728000223636627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,4095,0.01852799952030182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,2047,0.015386667102575302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,4095,0.01676799977819125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,8191,0.0284853329261144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,8191,0.022815999885400135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,16383,0.033957332372665405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,1,128,1,float16,fp8,16383,0.03481066723664602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1,0.012416000167528788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,3,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,7,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,3,0.011168000598748526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,15,0.012421333541472753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,7,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,15,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,31,0.012602667013804117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,63,0.012693333129088083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,31,0.011237333218256632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,63,0.011194666226704916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,127,0.012815999488035837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,127,0.011488000551859537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,255,0.015040000279744467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,255,0.012058666596810022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,511,0.015557333827018738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,511,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1023,0.016528000434239704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,1023,0.012383999923865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,2047,0.018063999712467194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,4095,0.025519999365011852
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,2047,0.013839999834696451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,4095,0.0169813334941864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,8191,0.030618667602539062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,8191,0.02316266546646754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,16383,0.04053333401679993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1,0.012383999923865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,2,128,1,float16,fp8,16383,0.0352906659245491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,3,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,7,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,3,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,7,0.011407999942700068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,15,0.012560000022252401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,31,0.01258133351802826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,15,0.011338666081428528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,31,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,63,0.012693333129088083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,63,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,127,0.012810666114091873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,255,0.01505600040157636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,127,0.011530666301647821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,511,0.015685333559910457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,255,0.01246400053302447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,511,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1023,0.017008000363906223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,2047,0.022463999688625336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,2047,0.0145066666106383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,1023,0.012874666601419449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,4095,0.02769600103298823
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,4095,0.017717332889636356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,8191,0.03766400118668874
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,8191,0.023919999599456787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,16383,0.204912006855011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,3,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,4,128,1,float16,fp8,16383,0.03643200049797694
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1,0.013248000293970108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,3,0.013455999394257864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,15,0.012506666282812754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,7,0.01312000056107839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,31,0.0124746672809124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,7,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,15,0.013183999806642532
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,127,0.012879999975363413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,31,0.013295999417702356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,63,0.013237333546082178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,255,0.015413332730531693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,63,0.01267733300725619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,127,0.013845333208640417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,255,0.014511999984582266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1023,0.021253332495689392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,511,0.015978666643301647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,511,0.014815999815861383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,1023,0.01591466615597407
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,2047,0.018330667167901993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,4095,0.04043733328580856
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,2047,0.026517334083716076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,4095,0.023658665517965954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,8191,0.20220800240834555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,8191,0.03363733241955439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,16383,0.3897706667582194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,64,8,128,1,float16,fp8,16383,0.1961066722869873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,1,0.015061333775520325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1,0.010581333190202713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,3,0.014959999670584997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,3,0.010698666175206503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,7,0.010757333288590113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,7,0.015125333021084467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,15,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,31,0.010858666151762009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,15,0.015087999403476715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,31,0.015072000523408255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,63,0.011045332998037338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,63,0.015370666980743408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,127,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,127,0.015893333901961643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,255,0.026394667724768322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,255,0.017279999951521557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,511,0.031162666777769726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,511,0.020538666596015293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1,0.01988799994190534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1023,0.04095466683308283
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,1,128,1,float16,fp8,1023,0.026687999566396076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,1,0.017136000096797943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,3,0.02000533292690913
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,3,0.017045332739750545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,7,0.020303999384244282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,7,0.01717866708834966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,15,0.020261333634455998
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,15,0.01720000058412552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,31,0.017231999586025875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,63,0.020848001043001812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,31,0.020682666450738907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,63,0.017616000026464462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,127,0.02202133337656657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,127,0.018543999642133713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,255,0.02701333413521449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,255,0.021312000850836437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,511,0.0367999995748202
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,511,0.02629866699377696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1,0.02685333291689555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1,0.023152001202106476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,3,0.0271519993742307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,2,128,1,float16,fp8,1023,0.0365226666132609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1023,0.20393067598342896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,3,0.023215999205907185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,7,0.02737066646416982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,15,0.027482666075229645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,15,0.023258666197458904
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,7,0.023130667706330616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,31,0.027621333797772724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,63,0.023792001108328503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,127,0.029877332349618275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,127,0.025653332471847534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,31,0.0232640008131663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,63,0.028143999477227528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,255,0.03979733337958654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,255,0.030400000512599945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1,0.04026666780312856
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,511,0.21132800976435342
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1,0.034501334031422935
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,511,0.04012800008058548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,3,0.034527999659379326
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,3,0.041135999063650765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,7,0.041519999504089355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,15,0.041573333243529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1023,0.4050399859746297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,4,128,1,float16,fp8,1023,0.20600533485412598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,31,0.0421973317861557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,31,0.034703999757766724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,63,0.042581334710121155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,7,0.03458133339881897
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,15,0.03461866577466329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,63,0.03513599932193756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1023,0.7833333015441895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,1023,0.38920533657073975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,255,0.2126506765683492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,127,0.0450133333603541
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,255,0.04604800045490265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,127,0.03583466758330663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,511,0.40676267941792804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,64,8,128,1,float16,fp8,511,0.21715199947357178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,1,0.015802666544914246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,1,0.020586666961510975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,3,0.02045866722861926
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,7,0.020538666596015293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,3,0.015647999942302704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,7,0.01587733378012975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,31,0.01616000011563301
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,15,0.020879998803138733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,15,0.015957333147525787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,31,0.02051199972629547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,63,0.016602666427691776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,63,0.020853333175182343
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,127,0.01786133274435997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,127,0.022128000855445862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,255,0.023168000082174938
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,1,0.03105599929889043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,1,128,1,float16,fp8,255,0.024495999018351238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,1,0.026709333062171936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,3,0.031343999008337654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,7,0.03160533308982849
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,3,0.026762666801611584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,7,0.026762666801611584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,15,0.03161599983771642
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,15,0.026496000587940216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,31,0.032085334261258446
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,31,0.026672000686327618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,63,0.032373333970705666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,63,0.027248000105222065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,127,0.0346666673819224
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,127,0.028933333853880566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,255,0.044266665975252785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,1,0.044549331068992615
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,2,128,1,float16,fp8,255,0.03367999941110611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,1,0.03839466720819473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,3,0.045370668172836304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,3,0.03835200021664301
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,7,0.0459146648645401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,7,0.03846399982770284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,15,0.03842133283615112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,15,0.04574400186538696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,31,0.04623466730117798
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,31,0.03858133405447006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,63,0.047135998805363975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,63,0.03902933249870936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,1,0.06922133266925812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,127,0.04957333207130432
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,127,0.040037333965301514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,1,0.05759466687838236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,255,0.22404267390569052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,4,128,1,float16,fp8,255,0.05000533163547516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,3,0.0705813318490982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,3,0.05784533421198527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,7,0.07147199908892314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,7,0.05798399945100149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,15,0.05795733133951823
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,15,0.07112533350785573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,63,0.07344533503055573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,31,0.057962665955225624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,31,0.07218666871388753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,63,0.058730666836102806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,127,0.22536534070968628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,127,0.06102933486302694
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,255,0.45842134952545166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,1,0.02565866708755493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,64,8,128,1,float16,fp8,255,0.21820799509684244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,1,0.033520000676314034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,3,0.025850666066010792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,3,0.033520000676314034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,15,0.03350933392842611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,7,0.025989333788553875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,7,0.03364799916744232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,31,0.033402666449546814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,31,0.0264533335963885
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,63,0.026943999032179516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,63,0.034128000338872276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,15,0.0259253333012263
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,127,0.029152000943819683
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,1,0.052522664268811546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,1,128,1,float16,fp8,127,0.03538133452335993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,1,0.04507199923197428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,3,0.05284800132115682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,3,0.045312002301216125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,7,0.0452159990866979
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,7,0.05323733389377594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,15,0.05362666646639506
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,31,0.054048001766204834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,15,0.045456002155939736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,31,0.04534933467706045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,63,0.054431999723116554
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,63,0.04604266583919525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,127,0.07432533303896587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,2,128,1,float16,fp8,127,0.0468800018231074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,1,0.07775466640790303
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,1,0.06648000081380208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,3,0.07891733447710673
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,3,0.06655466556549072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,7,0.07963733375072479
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,7,0.06666666766007741
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,15,0.07971199850241344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,15,0.06700799862543742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,31,0.08080533146858215
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,31,0.06702933212121327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,63,0.06748799979686737
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,63,0.08727467060089111
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,127,0.2545439998308818
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,1,0.12813333670298258
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,4,128,1,float16,fp8,127,0.07163733243942261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,1,0.10612266262372334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,3,0.12958932916323343
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,3,0.10570666193962097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,7,0.13241599996884665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,15,0.13192533453305563
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,7,0.10611200332641602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,31,0.13985066612561545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,15,0.10629866520563762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1,0.009130666653315226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,1,0.011333333949247995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,3,0.009258666386206945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,31,0.10635733604431152
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,3,0.011498666057984034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,7,0.009354666496316591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,7,0.01138666644692421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,15,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,31,0.00956266683836778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,31,0.011301333705584208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,63,0.009402666861812273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,63,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,127,0.009610666582981745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,127,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,255,0.013173333058754602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,511,0.013455999394257864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,255,0.01246400053302447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,15,0.009216000015536943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,1023,0.014159999787807465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,511,0.012565333396196365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1023,0.014501333236694336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,63,0.2737013300259908
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,2047,0.016677333662907284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,2047,0.01716800034046173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,4095,0.02327466756105423
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,63,0.10789866248766582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,4095,0.02457600086927414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,8191,0.03499199946721395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,16383,0.03958933303753535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,1,128,1,float16,fp8,16383,0.05907199780146281
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,3,0.013066666821638743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,3,0.011359999577204386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1,0.012847999731699625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,7,0.012954667210578918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,7,0.011359999577204386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,127,0.4580959876378377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,15,0.01138666644692421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,31,0.013162666310866674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,15,0.012944000462690989
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,64,8,128,1,float16,fp8,127,0.2481706738471985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,31,0.011338666081428528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,1,0.011498666057984034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,63,0.011503999431928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,63,0.01313599944114685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,255,0.015509333461523056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,127,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,8191,0.029477333029111225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,511,0.016250666230916977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,511,0.012847999731699625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,1023,0.0143306665122509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1023,0.01740266631046931
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,2047,0.023573334018389385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,4095,0.028698667883872986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,4095,0.023589332898457844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,2047,0.01728533332546552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,8191,0.0383093332250913
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,8191,0.03547733277082443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1,0.012815999488035837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,16383,0.059450666109720864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,3,0.012709333250919977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1,0.011578666667143503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,7,0.012917333592971167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,7,0.011557333171367645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,3,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,16383,0.20129066705703735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,15,0.012896000097195307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,15,0.011461333682139715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,31,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,31,0.011461333682139715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,127,0.013327999661366144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,63,0.011674666156371435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,63,0.01320533330241839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,127,0.011813333878914515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,255,0.015791999797026317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,255,0.013088000317414602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,511,0.016597333053747814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,2047,0.027045334378878277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,2,128,1,float16,fp8,255,0.012709333250919977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,127,0.013301332791646322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,2047,0.018144000321626663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,511,0.013493333011865616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,1023,0.01482133318980535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1023,0.021935999393463135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,4095,0.041082667807737984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,8191,0.036544000109036766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,4095,0.024346667031447094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,3,0.012885333349307379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,8191,0.20419732729593912
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,16383,0.39503999551137287
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,4,128,1,float16,fp8,16383,0.19325333833694458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1,0.013605333864688873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,3,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,7,0.012896000097195307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1,0.012800000607967377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,15,0.012954667210578918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,7,0.013471999516089758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,31,0.012954667210578918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,63,0.01313599944114685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,15,0.013637332866589228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,31,0.01341333364446958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,63,0.013541333377361298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,127,0.01360000049074491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,127,0.013909333695967993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1023,0.025957333544890087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,255,0.01634666696190834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,255,0.01526933287580808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,511,0.016229332735141117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,1023,0.01884799947341283
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,2047,0.03576533248027166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,2047,0.024090667565663654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,511,0.021168000996112823
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,4095,0.03410666684309641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,4095,0.20204265912373862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,8191,0.19523733854293823
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,8191,0.3896799882253011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,16383,0.7575360139211019
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,64,8,128,1,float16,fp8,16383,0.37382400035858154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,1,0.04460800190766653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,1,0.036101333796978
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,7,0.0455626646677653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,3,0.045141334335009255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,3,0.036415999134381614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,15,0.04535999894142151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,7,0.03616533428430557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,15,0.03607466568549474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,31,0.045968001087506614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,31,0.03622400015592575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,1,0.056186666091283165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,1,0.09362666805585225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,63,0.04650133351484934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,3,0.09411733349164327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,1,128,1,float16,fp8,63,0.03659199923276901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,3,0.05638400216897329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,7,0.09741866588592529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,7,0.056314667065938316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,15,0.09814400474230449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,31,0.1114453375339508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,15,0.05671466886997223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,31,0.05669866502285004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,63,0.17942400773366293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,2,128,1,float16,fp8,63,0.05807999769846598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,1,0.1442400018374125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,1,0.09778666496276855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,3,0.09834133585294087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,3,0.14726932843526205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,7,0.15450666348139444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,7,0.09817066788673401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,15,0.16577066977818808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,15,0.0983893374602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,31,0.19898132483164468
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,31,0.0986293355623881
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,63,0.3150613307952881
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,1,0.17672000328699747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,1,0.2474400003751119
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,4,128,1,float16,fp8,63,0.10761066277821858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,3,0.255621333916982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,3,0.17725332578023276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,7,0.17697066068649292
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,7,0.2759946584701538
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,15,0.300053338209788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,15,0.1773759921391805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,31,0.18266133467356363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,31,0.33450667063395184
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,63,0.5267733335494995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,64,8,128,1,float16,fp8,63,0.2779200077056885
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,3,0.1293226679166158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,3,0.06320000191529591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,7,0.12063999970753987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,1,0.12104533116022746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,15,0.14627200365066528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,1,0.06313066681226094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,15,0.06316799918810527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,7,0.06326400240262349
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,31,0.17688000202178955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,1,128,1,float16,fp8,31,0.06330666442712148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,1,0.26528533299763996
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,1,0.10306666294733684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,3,0.2744106650352478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,3,0.10327999790509541
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,7,0.2932586669921875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,7,0.10364799698193868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,15,0.2902719974517822
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,31,0.33419732252756756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,15,0.10382933417956035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,2,128,1,float16,fp8,31,0.10539733370145161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,1,0.3713173468907674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,1,0.18920532862345377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,3,0.3917653163274129
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,3,0.18997865915298462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,7,0.39427733421325684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,15,0.4076160192489624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,7,0.1902666687965393
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,15,0.19499200582504272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,31,0.46553067366282147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,4,128,1,float16,fp8,31,0.21608533461888632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,1,0.5879626671473185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,1,0.34802667299906415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,3,0.6195626656214396
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1,0.009418666362762451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,1,0.011754666765530905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,3,0.009370666618148485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,7,0.6424640019734701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,3,0.011711999773979187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,7,0.009338666374484697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,3,0.3484853506088257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,15,0.009338666374484697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,7,0.011887999872366587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,15,0.011722666521867117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,31,0.011855999628702799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,63,0.009648000200589498
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,7,0.35417068004608154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,63,0.011786667009194693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,127,0.009861333295702934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,31,0.009775999933481216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,127,0.01191466674208641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,255,0.013621332744757334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,15,0.6434079806009928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,511,0.014159999787807465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1023,0.01597333326935768
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,511,0.015034666905800501
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,1023,0.017903999735911686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,2047,0.023925334215164185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,4095,0.028351999819278717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,4095,0.03578133384386698
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,8191,0.038293334345022835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,15,0.36005866527557373
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,255,0.01358933374285698
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1,0.012831999609867731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,1,0.01173866664369901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,3,0.013088000317414602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,3,0.011813333878914515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,1,128,1,float16,fp8,8191,0.05973333120346069
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,7,0.013034666577974955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,15,0.013114667187134424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,2047,0.0230880007147789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,15,0.011706666400035223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,31,0.011839999506870905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,31,0.013157332936922709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,63,0.013269333789745966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,127,0.013408000270525614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,63,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,127,0.012026666353146235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,255,0.016122666498025257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,255,0.013669333110253016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,511,0.016965333372354507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,511,0.015119999647140503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1023,0.02279466638962428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,1023,0.018170667191346485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,7,0.011722666521867117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,4095,0.041850666205088295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,31,0.6940266291300455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,2047,0.02794666588306427
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,2047,0.02422933280467987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,4095,0.036389333506425224
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,8191,0.20153599977493286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,2,128,1,float16,fp8,8191,0.059978668888409935
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1,0.01293333371480306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,3,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1,0.011973333855470022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,3,0.012965332716703415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,7,0.012117333710193634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,15,0.013104000439246496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,31,0.013167999684810638
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,15,0.011936000237862269
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,31,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,63,0.013242666920026144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,63,0.012165332833925882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,127,0.013776000589132309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,127,0.012421333541472753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,255,0.014202666779359182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,255,0.016447999825080235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,511,0.02149333308140437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,511,0.015717333803574245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,2047,0.03643200049797694
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1023,0.026687999566396076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,2047,0.024901332954565685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,1023,0.01884799947341283
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,4095,0.03728000074625015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,4095,0.20427733659744263
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,8191,0.39371732870737713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,7,0.013104000439246496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1,0.01303999995191892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1,0.013967999567588171
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,3,0.013839999834696451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,64,8,128,1,float16,fp8,31,0.3924266497294108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,4,128,1,float16,fp8,8191,0.19413334131240845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,3,0.01313599944114685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,7,0.013866666704416275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,7,0.013365333278973898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,15,0.01393066719174385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,15,0.013232000172138214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,31,0.013274667163689932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,31,0.013951999445756277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,63,0.013477332890033722
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,127,0.014778666198253632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,255,0.021231998999913532
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,127,0.01413333291808764
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,255,0.016858667135238647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,511,0.026320000489552815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1023,0.03615466753641764
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,511,0.019498666127522785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,2047,0.20269334316253662
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,1023,0.02478933334350586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,2047,0.03472533325354258
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,4095,0.19641067584355673
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1,0.009317333499590555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,4095,0.39027734597524005
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,8191,0.3746986786524455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,8191,0.759231964747111
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,3,0.009434666484594345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,1,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,64,8,128,1,float16,fp8,63,0.014314666390419006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,3,0.012085333466529846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,7,0.009488000224033991
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,7,0.012063999970753988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,31,0.009482666850090027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,31,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,15,0.012074666718641916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,15,0.009455999980370203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,63,0.009594666461149851
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,127,0.00997866690158844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,63,0.012191999703645706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,255,0.013845333208640417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,255,0.014208000153303146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,127,0.012624000509579977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,511,0.01525866612792015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,511,0.01682666689157486
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1023,0.021557333568731945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,1023,0.022800001005331676
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1,0.013130666067202887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,2047,0.034874667723973594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,2047,0.026730666557947796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,3,0.013253333667914072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,4095,0.040378667414188385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,1,0.012122667084137598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,1,128,1,float16,fp8,4095,0.05862933397293091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,3,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,7,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,15,0.013167999684810638
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,7,0.012346666306257248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,31,0.013264000415802002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,63,0.012229333321253458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,127,0.014005333185195923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,63,0.013584000368913015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,127,0.012661332885424295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,31,0.012170666207869848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,15,0.012165332833925882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,255,0.01423466702302297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,255,0.017125333348910015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,511,0.022469334304332733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,511,0.017194667210181553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1023,0.02743999908367793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,4095,0.20098666350046793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,1023,0.023130667706330616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,4095,0.05922133227189382
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1,0.013354666531085968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,2047,0.03719999889532725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,2,128,1,float16,fp8,2047,0.035258665680885315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,3,0.01239466667175293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,7,0.013503999759753546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,15,0.013562666873137156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,7,0.012357333054145178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,3,0.013301332791646322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,15,0.012517333030700684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,63,0.013866666704416275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,31,0.013557333499193192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,63,0.01259200026591619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,127,0.014368000129858652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,31,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,255,0.014736000448465347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,127,0.013221333424250284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,255,0.021712000171343487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,511,0.02679466704527537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,511,0.017738666385412216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,2047,0.20468799273173013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,2047,0.036746665835380554
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1023,0.03640000025431315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,1023,0.023957334458827972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,4,128,1,float16,fp8,4095,0.19357866048812866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1,0.016976000120242436
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,4095,0.43057068188985187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,3,0.01444799949725469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1,0.014394666999578476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,3,0.017077332983414333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,15,0.017210666090250015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,7,0.017279999951521557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,15,0.014511999984582266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,7,0.014469332993030548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,31,0.014576000471909841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,63,0.017818666994571686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,31,0.017456000049908955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,127,0.018863999595244724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,63,0.014890667051076889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,127,0.015962666521469753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,255,0.023823998868465424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,255,0.018805333723624546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1023,0.2009333372116089
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,511,0.03367999941110611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,1023,0.033786666889985405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,511,0.023887999355793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,2047,0.38833598295847577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,1,0.010512000570694605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1,0.008703999842206636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,3,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,7,0.008656000097592672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,4095,0.3744106690088908
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,3,0.0086666668454806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,15,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,48,8,128,1,float16,fp8,2047,0.1962826649347941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,4095,0.7554826736450195
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,15,0.010608000059922537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,31,0.008858666444818178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,31,0.01055466632048289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,63,0.00892800030608972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,63,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,127,0.009045333291093508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,127,0.01089599976936976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,255,0.012485332787036896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,511,0.012805332740147909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,511,0.01156266654531161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,255,0.011722666521867117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1023,0.01349866638580958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,4095,0.016879999389251072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,1023,0.011605333536863327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,2047,0.014469332993030548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,4095,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,2047,0.01184533288081487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,8191,0.021541332205136616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,8191,0.013781332721312841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,16383,0.03091199944416682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1,0.008485333373149237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,1,128,1,float16,fp8,16383,0.016677333662907284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,3,0.008672000219424566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,7,0.008933333059151968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,15,0.008597333605090777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,7,0.010703999549150467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,3,0.010784000158309937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,15,0.010645333677530289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,63,0.008879999940594038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,31,0.008752000207702318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,31,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,63,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,127,0.008949333180983862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,127,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,255,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,511,0.012810666114091873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,255,0.011658667276302973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,511,0.011711999773979187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1023,0.013258667041858038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,1023,0.011637333780527115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,2047,0.014442666123310724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,4095,0.01682666689157486
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,2047,0.01190399999419848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,4095,0.012495999534924826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,8191,0.02184533327817917
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,16383,0.03543466577927271
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,8191,0.013807999591032663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1,0.00855466661353906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,2,128,1,float16,fp8,16383,0.017029333859682083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,3,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,7,0.008725333337982496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,15,0.008746666833758354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,7,0.010751999914646149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,31,0.008922666932145754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,15,0.010911999891201654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,31,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,63,0.008842666943868002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,127,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,255,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,63,0.010858666151762009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,127,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,255,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,511,0.012650666137536367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,511,0.011834666132926941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1023,0.013359999905029932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,2047,0.014736000448465347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,4095,0.01735466718673706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,1023,0.012015999605258306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,2047,0.01211200033624967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,4095,0.012885333349307379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,8191,0.026367999613285065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,16383,0.03152533372243246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,8191,0.014389333625634512
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1,0.008597333605090777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,4,128,1,float16,fp8,16383,0.017711999515692394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,3,0.008757333581646284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1,0.011381333072980246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,15,0.008879999940594038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,3,0.010965333630641302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,7,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,7,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,15,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,63,0.008890666688481966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,31,0.011274666835864386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,63,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,127,0.009119999905427298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,255,0.012426666915416718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,127,0.01128000020980835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,31,0.00879466657837232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,511,0.012938667088747025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,255,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1023,0.013530666629473368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,511,0.013914667069911957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,1023,0.014021333307027817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,2047,0.014965333044528961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,4095,0.02186666677395503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,2047,0.01461333284775416
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,4095,0.015824000040690105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,8191,0.026757332185904186
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,8191,0.0183999997874101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,16383,0.03661333272854487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,48,8,128,1,float16,fp8,16383,0.023775999744733173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1,0.00874133345981439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,1,0.010757333288590113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,3,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,7,0.008874666566650072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,15,0.008885333314538002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,15,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,31,0.008954666554927826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,7,0.010661333799362183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,3,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,31,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,63,0.009088000282645226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,127,0.009296000003814697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,63,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,127,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,255,0.01249066616098086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,255,0.01163200040658315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,511,0.013002666334311167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,511,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,1023,0.011882666498422623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1023,0.013546666751305262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,2047,0.014767999450365702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,2047,0.01209066684047381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,4095,0.017055999487638474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,8191,0.022650666534900665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,4095,0.013541333377361298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,8191,0.01674666628241539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,16383,0.03677866607904434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,1,128,1,float16,fp8,16383,0.022709332406520844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1,0.011989332735538483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,3,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,1,0.010746666540702185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,15,0.012117333710193634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,7,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,7,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,15,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,31,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,127,0.012533333152532578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,31,0.010922666639089584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,63,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,127,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,255,0.01479999969402949
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,63,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,255,0.011781333635250727
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,511,0.015157333264748255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1023,0.015930666277805965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,511,0.011749333391586939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,2047,0.017114666601022083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,1023,0.01201066623131434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,4095,0.01995733380317688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,2047,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,4095,0.013877333452304205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,8191,0.029482667644818623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,8191,0.016735999534527462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,16383,0.034389334420363106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,2,128,1,float16,fp8,16383,0.023013333479563396
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1,0.012063999970753988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,3,0.012202666451533636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1,0.011120000233252844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,7,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,15,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,3,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,7,0.010938666760921478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,31,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,63,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,127,0.012608000387748083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,63,0.011087999989589056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,255,0.014842666685581207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,127,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,511,0.015157333264748255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,255,0.012069333344697952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,511,0.011962667107582092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1023,0.016000000139077503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,2047,0.017551999539136887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,1023,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,4095,0.024458666642506916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,2047,0.012655999511480331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,8191,0.029626667499542236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,4095,0.014373333503802618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,8191,0.017456000049908955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,16383,0.03922666609287262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,4,128,1,float16,fp8,16383,0.023685333629449207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1,0.012272000312805176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,7,0.012223999947309494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,7,0.01139733319481214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,15,0.012373333175977072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,3,0.012245333443085352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,3,0.011413333316644033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,15,0.011333333949247995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,31,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,31,0.01173866664369901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,63,0.011861333002646765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,63,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,127,0.012736000120639801
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,255,0.014965333044528961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,511,0.01524266724785169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,127,0.012602667013804117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,255,0.013999999811251959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,2047,0.02216533323129018
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,511,0.014261333892742792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,2047,0.015386667102575302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1023,0.01646399994691213
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,1023,0.014602666099866232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,4095,0.018207999567190807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,8191,0.03711999952793121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,4095,0.027072000006834667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,8191,0.023423999547958374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,16383,0.20228799184163412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,48,8,128,1,float16,fp8,16383,0.033386667569478355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1,0.00972800018886725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,1,0.012981332838535309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,3,0.009824000298976898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,3,0.013007999708255133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,7,0.009685333197315535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,7,0.012863999853531519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,15,0.009813333551088968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,15,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,31,0.010112000008424124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,31,0.012853333105643591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,63,0.010079999764760336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,63,0.013082666943470636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,127,0.010490667074918747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,127,0.013237333546082178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,255,0.016069332758585613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,255,0.014842666685581207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,511,0.021962667504946392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,511,0.017946666727463405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1023,0.026928000152111053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,1023,0.023765332996845245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1,0.014074667046467463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,2047,0.037717332442601524
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,1,0.012960000584522883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,1,128,1,float16,fp8,2047,0.03606933355331421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,7,0.014149333039919535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,3,0.01301866645614306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,3,0.01404800017674764
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,7,0.013130666067202887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,31,0.01421333352724711
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,15,0.014511999984582266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,15,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,63,0.013232000172138214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,63,0.014522666732470194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,31,0.013189333180586496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,127,0.015173333386580149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,127,0.013834666460752487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,255,0.023749334116776783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,255,0.015487999965747198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,511,0.01841066653529803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,511,0.028751999139785767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1,0.017605333278576534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1023,0.03847466657559077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,1023,0.02459733436505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1,0.015189333508412043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,3,0.017840000490347546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,3,0.015050667027632395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,2,128,1,float16,fp8,2047,0.03743999948104223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,7,0.017903999735911686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,2047,0.20451200008392334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,15,0.01522133375207583
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,7,0.01505600040157636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,31,0.018090666582187016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,15,0.017893332988023758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,31,0.015141333142916361
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,63,0.015626666446526844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,127,0.01658133293191592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,63,0.018592000007629395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,255,0.01931200052301089
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,255,0.02456533412138621
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,127,0.019530666371186573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,511,0.024351999163627625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,511,0.034245334565639496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,2047,0.39213331540425617
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1023,0.20383999745051065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,2047,0.19932266076405844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,4,128,1,float16,fp8,1023,0.034832000732421875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1,0.021136000752449036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1,0.024688000480333965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,3,0.021045332153638203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,7,0.020981334149837494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,3,0.02516799916823705
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,15,0.021146667500336964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,31,0.025455998877684276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,7,0.025616000096003216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,31,0.021253332495689392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,63,0.021930667261282604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,63,0.025989333788553875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,127,0.0278613343834877
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,127,0.023018665611743927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,15,0.025439999997615814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,255,0.03730666637420654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,255,0.028181334336598713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1023,0.3935360113779704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,1023,0.19789334138234457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,2047,0.7626187006632487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,511,0.2044693430264791
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,1,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,3,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,511,0.03772799919048945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,3,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1,0.008863999818762144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,7,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,15,0.00890666681031386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,48,8,128,1,float16,fp8,2047,0.37882665793100995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,31,0.009082666908701261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,63,0.009162666896979014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,31,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,63,0.011130666981140772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,127,0.009338666374484697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,127,0.011338666081428528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,255,0.012085333466529846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,255,0.012757333616415659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,511,0.011839999506870905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1023,0.013690666606028875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,511,0.01321600005030632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,7,0.008992000172535578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,2047,0.01515199989080429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,1023,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,15,0.011087999989589056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,2047,0.013738666971524557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,4095,0.018170667191346485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,8191,0.027823999524116516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,8191,0.02276800076166789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,4095,0.016544000556071598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,16383,0.03305066625277201
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,1,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,1,128,1,float16,fp8,16383,0.03446399917205175
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,3,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,3,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,7,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,15,0.012517333030700684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,7,0.012650666137536367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,15,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,31,0.012442667037248611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,63,0.012714666624863943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,31,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,127,0.012960000584522883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,63,0.011237333218256632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,127,0.011482667177915573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,255,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,511,0.015466666469971338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,255,0.015050667027632395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1023,0.01629866659641266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,511,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,2047,0.017935999979575474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,1023,0.01240533341964086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,2047,0.013904000322024027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,4095,0.02496533344189326
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,8191,0.03046400099992752
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,4095,0.017194667210181553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,16383,0.04009066770474116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,8191,0.023103999594847362
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1,0.012309333930412928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,3,0.012373333175977072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,2,128,1,float16,fp8,16383,0.03486400097608566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,7,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,3,0.011274666835864386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,15,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,31,0.012453333785136541
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,15,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,63,0.012698666503032049
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,31,0.011333333949247995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,63,0.01128000020980835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,255,0.01498666654030482
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,127,0.01156266654531161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,511,0.015647999942302704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,127,0.012869333227475485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,255,0.012223999947309494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,511,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1023,0.016666666915019352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,2047,0.022453332940737408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,1023,0.012778667112191519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,2047,0.014474666366974512
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,4095,0.017322666943073273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,8191,0.0373333344856898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,4095,0.027802666028340656
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,8191,0.023599999646345775
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,16383,0.20439465840657553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1,0.012426666915416718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,4,128,1,float16,fp8,16383,0.03605866680542628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,7,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,3,0.013237333546082178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1,0.01312000056107839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,15,0.012629333883523941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,3,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,31,0.012458667159080505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,7,0.012970666090647379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,15,0.01310933381319046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,31,0.013269333789745966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,63,0.013088000317414602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,127,0.013642666240533194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,127,0.012874666601419449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,63,0.012655999511480331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,255,0.015274666249752045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,255,0.014485333114862442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1023,0.02126399924357732
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,511,0.016250666230916977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,511,0.014848000059525171
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,4095,0.040549332896868386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,2047,0.02638400097688039
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,1023,0.015573333948850632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,4095,0.023402666052182514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,2047,0.018383999665578205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,8191,0.03365866591533025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,16383,0.3882826566696167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,8191,0.22115200757980347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,48,8,128,1,float16,fp8,16383,0.19454934199651083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1,0.010064000263810158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,1,0.014250667144854864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,3,0.010191999996701876
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,3,0.014352000008026758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,7,0.014293332894643148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,7,0.010144000252087912
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,15,0.010245333115259806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,15,0.014271999398867289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,31,0.010357333347201347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,31,0.014138666292031607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,63,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,63,0.014511999984582266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,127,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,127,0.014949332922697067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,255,0.024069334069887798
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,255,0.01655999943614006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,511,0.029274667302767437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,511,0.019679999599854153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1,0.01874133323629697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1023,0.03938666731119156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,1,128,1,float16,fp8,1023,0.025642665723959606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,3,0.01899733394384384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,3,0.01624533285697301
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,1,0.016522667060295742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,7,0.01899733394384384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,15,0.01907733331123988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,7,0.01637866720557213
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,15,0.01639466608564059
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,31,0.01934933289885521
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,31,0.016421332955360413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,63,0.019632000476121902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,63,0.01681600014368693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,127,0.02073066681623459
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,127,0.017717332889636356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,255,0.025781333446502686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,255,0.020560000091791153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,511,0.03562133262554804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,511,0.025594666600227356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1,0.025770666698614757
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1,0.022197333474953968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1023,0.20011200507481894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,3,0.02613866577545802
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,2,128,1,float16,fp8,1023,0.03606933355331421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,3,0.022453332940737408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,7,0.026416001220544178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,15,0.02638400097688039
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,7,0.022384000321229298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,15,0.022287999590237934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,31,0.02242133269707362
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,63,0.02311466634273529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,127,0.0288426677385966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,31,0.026650667190551758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,127,0.024330665667851765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,63,0.027141332626342773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,255,0.03884266565243403
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,255,0.02940800040960312
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1,0.03962666789690653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,511,0.0391893337170283
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,511,0.2085226575533549
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1,0.033999999364217125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,3,0.040133332212766014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1023,0.40167999267578125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,3,0.033861334125200905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,4,128,1,float16,fp8,1023,0.20304532845815024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,15,0.04088533421357473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,7,0.04079466561476389
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,15,0.0340693344672521
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,7,0.03402666747570038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,31,0.03401066611210505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,63,0.03440000116825104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,127,0.04429866870244344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,127,0.034927998979886375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,63,0.04181333382924398
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,31,0.041178666055202484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,255,0.04513066510359446
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1023,0.7757653395334879
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,255,0.22733332713445029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,1023,0.3853013515472412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,48,8,128,1,float16,fp8,511,0.20274666945139566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,511,0.40200531482696533
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,1,0.01488000030318896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,1,0.018842666099468868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,3,0.018874666343132656
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,7,0.018885333091020584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,3,0.014767999450365702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,31,0.015413332730531693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,7,0.015087999403476715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,31,0.01899733394384384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,63,0.015583999454975128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,15,0.018965333700180054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,63,0.019343999524911244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,15,0.01515199989080429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,127,0.016927999754746754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,127,0.02015999952952067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,255,0.02197866638501485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,1,128,1,float16,fp8,255,0.022997332115968067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,1,0.028922667105992634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,1,0.02499199906984965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,3,0.029120000700155895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,3,0.025072000920772552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,7,0.029333333174387615
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,7,0.024906667570273083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,15,0.02959999938805898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,15,0.02497066557407379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,31,0.029818666477998097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,63,0.030165334542592365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,31,0.025311999022960663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,63,0.02550400048494339
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,127,0.03214933226505915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,127,0.026965332527955372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,255,0.04237333436806997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,1,0.04283200204372406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,2,128,1,float16,fp8,255,0.032357332607110344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,1,0.03681066632270813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,3,0.04353066782156626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,3,0.03691199918588003
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,7,0.04390933116277059
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,7,0.036661334335803986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,15,0.0367999995748202
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,31,0.04452266792456309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,15,0.04399999976158142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,31,0.036864000062147774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,63,0.04527466495831808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,63,0.03718933214743932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,127,0.04771733283996582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,1,0.0674186646938324
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,1,0.055957332253456116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,127,0.03852266569932302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,255,0.21864533424377441
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,4,128,1,float16,fp8,255,0.048800001541773476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,3,0.0689333329598109
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,3,0.0561706672112147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,7,0.06929600238800049
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,15,0.06942399839560191
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,7,0.05619733532269796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,63,0.0573226660490036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,63,0.07190933326880138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,15,0.05602666735649109
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,31,0.05629866818586985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,31,0.07011199990908305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,127,0.2185653249422709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,127,0.058490668733914696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,48,8,128,1,float16,fp8,255,0.21261332432428995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,255,0.41493332386016846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,1,0.023984000086784363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,1,0.03030399978160858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,3,0.030192000170548756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,3,0.024186665813128155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,7,0.024336000283559162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,31,0.025008000433444977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,31,0.030218665798505146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,7,0.03011200080315272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,63,0.025349333882331848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,63,0.030965333183606465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,15,0.024442667762438457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,15,0.030415999392668407
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,127,0.027306665976842243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,1,0.048485333720842995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,1,0.04201066493988037
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,1,128,1,float16,fp8,127,0.03259733319282532
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,3,0.04892266790072123
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,3,0.04223999877770742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,7,0.04966400067011515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,7,0.042090664307276406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,15,0.04953599969546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,15,0.04207466542720795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,31,0.050330668687820435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,31,0.042223999897638954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,63,0.05124266445636749
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,63,0.042917331059773765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,1,0.0738506664832433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,1,0.06313600142796834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,127,0.05505600074927012
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,2,128,1,float16,fp8,127,0.04420800010363261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,3,0.07550933460394542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,7,0.07635200023651123
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,3,0.06323199967543285
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,7,0.06351466476917267
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,15,0.07603733241558075
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,15,0.06322133541107178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,31,0.0636053333679835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,31,0.07700799902280171
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,63,0.07878399888674419
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,63,0.06409599880377452
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,127,0.26023467381795246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,4,128,1,float16,fp8,127,0.06676800052324931
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,1,0.1249066690603892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,1,0.10236799716949463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,3,0.12657066186269125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,3,0.10301867127418518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,7,0.12889599800109863
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,7,0.10268266995747884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,15,0.12837333480517069
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,15,0.10320533315340678
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,1,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,31,0.1323093374570211
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,3,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,31,0.10309333602587382
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1,0.008992000172535578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,3,0.011322667201360067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,7,0.009216000015536943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,15,0.011183999478816986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,31,0.009301333377758661
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,31,0.01138666644692421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,63,0.24030399322509766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,63,0.009269333134094873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,63,0.011231999844312668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,15,0.009098666409651438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,127,0.009514666472872099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,63,0.1043893297513326
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,127,0.01145600030819575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,255,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,511,0.013274667163689932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,511,0.012458667159080505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,1023,0.013914667069911957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,2047,0.01695466662446658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,4095,0.023498666783173878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,2047,0.01591466615597407
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,4095,0.02295999974012375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,16383,0.038506666819254555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,8191,0.02881066749493281
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,8191,0.03477333237727483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,1,128,1,float16,fp8,16383,0.05888533095518748
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,127,0.4427146514256795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,3,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,1,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,3,0.011237333218256632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1023,0.014138666292031607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,255,0.013013333082199097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,7,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,15,0.011333333949247995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,48,8,128,1,float16,fp8,127,0.25204267104466754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,63,0.012842666357755661
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,31,0.011370666325092316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,15,0.012768000364303589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,31,0.012837332983811697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,63,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,127,0.013114667187134424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,255,0.01259200026591619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,255,0.015210667004187902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,511,0.01588800052801768
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1023,0.016997333616018295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,511,0.012736000120639801
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,1023,0.014138666292031607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,2047,0.02292799949645996
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,4095,0.028255999088287354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,8191,0.03815466662247976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,4095,0.02319466571013133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,2047,0.01724799970785777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,7,0.012666666259368261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,8191,0.03530666728814443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,16383,0.059061333537101746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1,0.011333333949247995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1,0.01259200026591619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,16383,0.20061333974202475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,3,0.011584000041087469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,7,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,3,0.012650666137536367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,15,0.012805332740147909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,7,0.012778667112191519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,31,0.012800000607967377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,15,0.011503999431928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,63,0.012757333616415659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,31,0.011461333682139715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,63,0.011381333072980246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,2,128,1,float16,fp8,127,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,255,0.01553600033124288
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,127,0.011855999628702799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,127,0.013072000195582708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,255,0.012655999511480331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,511,0.012997332960367203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1023,0.02161066730817159
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,2047,0.026821332673231762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,511,0.016336000214020412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,2047,0.017781333376963932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,1023,0.014725333700577417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,4095,0.040693332751592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,4095,0.023962666591008503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,8191,0.036490666369597115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,8191,0.20333333810170492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,4,128,1,float16,fp8,16383,0.19316800435384116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1,0.01341333364446958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,3,0.01267733300725619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1,0.012730666746695837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,16383,0.39211734135945636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,3,0.013365333278973898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,7,0.013424000392357508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,7,0.012757333616415659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,15,0.012752000242471695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,15,0.01331199953953425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,31,0.013056000073750814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,63,0.013066666821638743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,31,0.013338666409254074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,127,0.013327999661366144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,255,0.016021333634853363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,511,0.020981334149837494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,127,0.013957332819700241
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,255,0.015135999768972397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,63,0.01351999988158544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,511,0.015840000162522
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,2047,0.035802667339642845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,1023,0.018624000251293182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1023,0.0259253333012263
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,4095,0.20174400011698404
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,2047,0.02378133436044057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,4095,0.03385599950949351
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,8191,0.1950719952583313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,48,8,128,1,float16,fp8,16383,0.3733280102411906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,8191,0.42918399969736737
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,16383,0.7559146881103516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,1,0.04189866781234741
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,1,0.034661332766215004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,3,0.042266666889190674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,7,0.04262933135032654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,7,0.034490667283535004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,3,0.03475199888149897
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,15,0.04275199770927429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,15,0.03465066601832708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,31,0.04346133271853129
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,31,0.03465600063403448
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,1,0.08501332998275757
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,63,0.04380266865094503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,1,0.054976001381874084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,1,128,1,float16,fp8,63,0.034789333740870156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,3,0.08612266182899475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,3,0.05492799977461497
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,7,0.08742933471997578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,15,0.08689066767692566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,7,0.054832001527150474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,31,0.05513066550095876
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,63,0.14460266629854837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,31,0.08905067046483357
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,63,0.05637866755326589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,2,128,1,float16,fp8,15,0.05506666501363119
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,1,0.13753066460291544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,1,0.0958026647567749
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,3,0.13921067118644714
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,3,0.09622400005658467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,7,0.14102400342623392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,7,0.09663466612497966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,15,0.14341866970062256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,15,0.09633066256841023
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,31,0.16869866847991943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,31,0.096778670946757
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,63,0.2850506703058879
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,4,128,1,float16,fp8,63,0.09888000289599101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,1,0.23979200919469199
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,1,0.17550933361053467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,3,0.24409600098927817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,3,0.17516799767812094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,7,0.25140267610549927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,7,0.17512534062067667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,15,0.17569067080815634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,15,0.2765386700630188
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,31,0.3110026717185974
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,31,0.17871467272440592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,63,0.49423468112945557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,48,8,128,1,float16,fp8,63,0.2633333404858907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,1,0.0749013324578603
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,1,0.05961599946022034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,3,0.07619200150171916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,3,0.059664001067479454
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,7,0.07764799892902374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,7,0.05986666679382324
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,15,0.08577066659927368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,15,0.06048533320426941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,31,0.11084799965222676
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,1,128,1,float16,fp8,31,0.05992533266544342
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,1,0.2019946575164795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,1,0.10001599788665771
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,3,0.2111253341039022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,3,0.10037333766619365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,7,0.22204800446828207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,7,0.10048533479372661
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,15,0.2369920015335083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,15,0.10052800178527832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,31,0.2664960026741028
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,2,128,1,float16,fp8,31,0.10089066624641418
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,1,0.314517339070638
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,1,0.18571732441584268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,3,0.3400426705678304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,3,0.18609066804250082
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,7,0.3482026656468709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,7,0.1857866644859314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,15,0.35577599207560223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,15,0.1869706710179647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,31,0.4047520160675049
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,4,128,1,float16,fp8,31,0.20206934213638306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,1,0.549237330754598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,1,0.3436853488286336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,3,0.5711893240610758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1,0.009285333255926767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,1,0.011605333536863327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,3,0.009226666763424873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,3,0.34373335043589276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,3,0.011509332805871964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,7,0.009232000137368837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,7,0.5917919874191284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,15,0.009103999783595404
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,7,0.011557333171367645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,15,0.011605333536863327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,31,0.009183999771873156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,63,0.009578666960199675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,63,0.011525332927703857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,127,0.009621333330869675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,127,0.011525332927703857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,255,0.01321600005030632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,511,0.013760000467300415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,255,0.013066666821638743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,31,0.011589333415031433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,511,0.014688000082969666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1023,0.015184000134468079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,1023,0.01754666616519292
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,7,0.34563732147216797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,2047,0.022096000611782074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,2047,0.023503998915354412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,4095,0.02712533374627431
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,4095,0.03568000098069509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,1,128,1,float16,fp8,8191,0.059445331494013466
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1,0.012682666381200155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,1,0.011658667276302973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,3,0.011546666423479715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,3,0.012890666723251343
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,7,0.012917333592971167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,8191,0.037205333511034645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,7,0.011514666179815928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,15,0.012810666114091873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,31,0.013002666334311167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,15,0.01163200040658315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,31,0.011567999919255575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,63,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,127,0.013493333011865616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,63,0.011727999895811081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,127,0.011823999385039011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,255,0.015775999675194424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,15,0.5906026760737101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,255,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,511,0.014922666052977243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,511,0.016666666915019352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1023,0.02257599929968516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,1023,0.018031999468803406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,2047,0.023914667467276256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,4095,0.041509332756201424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,4095,0.035749333600203194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,8191,0.19977599382400513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1,0.011711999773979187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,2,128,1,float16,fp8,8191,0.05978133281071981
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1,0.012869333227475485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,3,0.01293333371480306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,15,0.35359466075897217
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,3,0.011749333391586939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,15,0.012965332716703415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,7,0.011829332758982977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,7,0.01302933320403099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,31,0.01302933320403099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,15,0.011701333026091257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,2047,0.027727998793125153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,31,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,63,0.013141332815090815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,63,0.011850666254758835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,127,0.013760000467300415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,255,0.016224000602960587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,255,0.013594667116800943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,127,0.012229333321253458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,31,0.6437493165334066
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,511,0.021482666333516438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,2047,0.036357333262761436
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1023,0.02657066782315572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,511,0.015285332997639975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,1023,0.018272000054518383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,2047,0.024703999360402424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,4095,0.20333866278330484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,8191,0.1938506762186686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,8191,0.3919893503189087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1,0.012975999464591345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,3,0.013162666310866674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,3,0.013679999858140945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,7,0.013088000317414602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,7,0.013888000200192133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,15,0.01310933381319046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,31,0.013381333400805792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,4,128,1,float16,fp8,4095,0.03735466549793879
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,15,0.013888000200192133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,48,8,128,1,float16,fp8,31,0.37674665451049805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,31,0.013765333841244379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,63,0.01351999988158544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,63,0.013898666948080063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,127,0.014096000542243322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,127,0.014639999717473984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,255,0.021157334248224895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1023,0.035930665830771126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,255,0.01647466669480006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,511,0.019194666296243668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,2047,0.20193066199620566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,511,0.02605866640806198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,2047,0.03457066665093104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,4095,0.38891200224558514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,4095,0.1959786613782247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,8191,0.3734240134557088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,8191,0.7576533158620199
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,1,0.012047999848922094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,3,0.01190399999419848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,3,0.009338666374484697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1,0.00922133338948091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,7,0.009375999992092451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,15,0.009472000102202097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,7,0.011829332758982977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,15,0.011877333124478659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,31,0.011999999483426413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,31,0.009525333220760027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,48,8,128,1,float16,fp8,1023,0.02476266771554947
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,63,0.009578666960199675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,127,0.009808000177145004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,63,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,127,0.012122667084137598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,255,0.013845333208640417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,255,0.013717333475748697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,511,0.016805333395799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,511,0.014794666320085526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1023,0.020874666670958202
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,2047,0.025914666553338368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,2047,0.03456533451875051
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,1023,0.022709332406520844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,4095,0.039813332259655
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,1,0.011962667107582092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,3,0.013359999905029932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,1,128,1,float16,fp8,4095,0.0584853341182073
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1,0.013151999562978745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,7,0.012042666474978128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,7,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,3,0.012063999970753988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,15,0.01192533348997434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,15,0.013370666652917862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,31,0.013424000392357508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,63,0.013541333377361298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,63,0.012234666695197424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,127,0.013845333208640417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,127,0.012448000411192576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,255,0.016901332885026932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,255,0.014064000298579534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,31,0.011999999483426413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,511,0.022309333086013794
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,511,0.016997333616018295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1023,0.027285332481066387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,1023,0.022997332115968067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,4095,0.058874666690826416
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,4095,0.19975467522939047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,2047,0.03702399879693985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,2,128,1,float16,fp8,2047,0.03504000107447306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1,0.013306666165590286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,7,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,3,0.013493333011865616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,7,0.013370666652917862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,3,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,15,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,31,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,15,0.013306666165590286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,63,0.013839999834696451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,31,0.013455999394257864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,127,0.014250667144854864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,63,0.012560000022252401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,127,0.012938667088747025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,255,0.0215786670645078
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,255,0.014453332871198654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,511,0.026687999566396076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,511,0.01758933315674464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1023,0.036373332142829895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,2047,0.03649600098530451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,1023,0.02386666586001714
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,2047,0.22237332661946616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,4095,0.39112532138824463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1,0.016778666526079178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,4,128,1,float16,fp8,4095,0.19369065761566162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,3,0.017125333348910015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,7,0.017114666601022083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,7,0.014335999886194864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,15,0.017114666601022083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,15,0.014314666390419006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1,0.01422400027513504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,3,0.014192000031471252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,31,0.014298666268587112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,31,0.01729600007335345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,63,0.014730667074521383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,63,0.01746133342385292
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,127,0.01886933296918869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,127,0.01565333331624667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,255,0.023973333338896435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,1023,0.03431999931732813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,255,0.018672000616788864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1023,0.21923200289408365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,2047,0.3868853251139323
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,511,0.03346666693687439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,511,0.0235359991590182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,1,0.010640000303586325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1,0.008597333605090777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,7,0.008736000085870424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,4095,0.7525920073191324
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,2047,0.196015993754069
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,7,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,3,0.008810666700204214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,3,0.010538666198650995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,40,8,128,1,float16,fp8,4095,0.37428800264994305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,15,0.00891733355820179
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,15,0.010682666053374609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,31,0.008896000062425932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,63,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,127,0.009002666920423508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,31,0.010741333166758219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,63,0.009002666920423508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,255,0.012495999534924826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,127,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,255,0.011578666667143503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,511,0.012762666990359625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1023,0.013327999661366144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,511,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,2047,0.014426667243242264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,1023,0.01173866664369901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,2047,0.011839999506870905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,4095,0.016693333784739178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,4095,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,8191,0.013568000247081121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,16383,0.03130666663249334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,8191,0.021418665846188862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,1,128,1,float16,fp8,16383,0.016650666793187458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1,0.0086666668454806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,1,0.010687999427318573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,3,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,3,0.008527999743819237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,7,0.008810666700204214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,7,0.010640000303586325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,15,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,31,0.008799999952316284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,15,0.008869333192706108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,63,0.008826666822036108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,31,0.0107893335322539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,127,0.009093333035707474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,255,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,127,0.010890666395425797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,511,0.012645332763592402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,63,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,255,0.011685332904259363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,511,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,1023,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,2047,0.014362666755914688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,2047,0.011829332758982977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,4095,0.01693333312869072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,8191,0.021829334398110706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,4095,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1023,0.013189333180586496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,8191,0.01393066719174385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,16383,0.035386666655540466
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1,0.0085333331177632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,2,128,1,float16,fp8,16383,0.017125333348910015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,3,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1,0.010794666906197866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,3,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,15,0.008816000074148178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,7,0.010837333897749582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,31,0.008912000184257826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,15,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,7,0.008912000184257826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,63,0.008912000184257826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,31,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,127,0.009141333401203156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,63,0.010863999525705973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,255,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,127,0.010970667004585266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,511,0.0124746672809124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,255,0.011578666667143503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,511,0.01201066623131434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1023,0.013290667285521826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,2047,0.014773332824309668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,1023,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,4095,0.01725333308180173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,2047,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,4095,0.01292266696691513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,8191,0.026362667481104534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,8191,0.01463466634353002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,16383,0.03153600047032038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,4,128,1,float16,fp8,16383,0.017535999417304993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,7,0.008938666433095932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1,0.008570666735370954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,3,0.011029332876205444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,15,0.008858666444818178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,7,0.011424000064531961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,3,0.008762666955590248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,31,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,127,0.009178666397929192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,15,0.01129066695769628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,63,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,63,0.008976000050703684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,31,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,127,0.01191466674208641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,255,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,511,0.01267733300725619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,255,0.01209066684047381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,511,0.013866666704416275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1023,0.013514666507641474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,2047,0.015130666395028433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,4095,0.021914665897687275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,8191,0.02677333354949951
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,1023,0.014106666048367819
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,2047,0.0145066666106383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,16383,0.036687999963760376
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,4095,0.015813333292802174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,8191,0.018464000274737675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,40,8,128,1,float16,fp8,16383,0.023621333142121632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,1,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,3,0.008757333581646284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,7,0.008954666554927826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1,0.008682666967312494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,15,0.008869333192706108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,7,0.011007999380429586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,3,0.010954666882753372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,15,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,31,0.008826666822036108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,31,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,63,0.008965333302815756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,127,0.009088000282645226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,63,0.010970667004585266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,127,0.011018666128317514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,255,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,255,0.011754666765530905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,511,0.012837332983811697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,1023,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,511,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1023,0.013445333888133367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,2047,0.014762666076421738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,2047,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,4095,0.0169813334941864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,4095,0.013653332988421122
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,8191,0.01659199967980385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,8191,0.02207999924818675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,1,128,1,float16,fp8,16383,0.022815999885400135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,16383,0.036415999134381614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,3,0.010928000013033548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,3,0.012282667060693106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,7,0.01211200033624967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,7,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,15,0.012165332833925882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,15,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,31,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,63,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,127,0.012522666404644648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,127,0.011130666981140772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,255,0.014901333798964819
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,511,0.011823999385039011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1023,0.01581866666674614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,1023,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,2047,0.01711999997496605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,511,0.015066667149464289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,2047,0.012357333054145178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,4095,0.019823999454577763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,8191,0.029301332930723827
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,4095,0.013770667215188345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,16383,0.034517332911491394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,8191,0.016773333152135212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1,0.012234666695197424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,2,128,1,float16,fp8,16383,0.0229066660006841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,3,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,3,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,7,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,15,0.012229333321253458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,7,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,31,0.012362666428089142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,63,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,31,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,127,0.012597333639860153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,63,0.011125333607196808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,15,0.01102399950226148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,127,0.011274666835864386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,255,0.014975999792416891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,255,0.011946666985750198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1023,0.01607999950647354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,2047,0.017653333644072216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,511,0.012154666086037954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,511,0.015168000012636185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,1023,0.01211200033624967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,2047,0.012965332716703415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,4095,0.024421334266662598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,8191,0.029637334247430164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,4095,0.01452800010641416
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,8191,0.017594666530688603
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,16383,0.03941866755485535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,3,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1,0.0124746672809124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,4,128,1,float16,fp8,16383,0.02369066576162974
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1,0.011418666690587997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,7,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,3,0.011157333850860596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,15,0.012309333930412928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,63,0.012421333541472753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,7,0.011701333026091257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,31,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,15,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,63,0.011674666156371435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,31,0.012229333321253458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,127,0.012538666526476542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,127,0.011909333368142446
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,255,0.014032000054915747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,255,0.01498666654030482
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,511,0.015402667224407196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,1023,0.014533333480358124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,511,0.014208000153303146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1023,0.01643199970324834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,2047,0.022218666970729828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,8191,0.036901332437992096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,2047,0.015471999843915304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,4095,0.027263998985290527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,4095,0.018394666413466137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,16383,0.202239990234375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,8191,0.02367999901374181
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,40,8,128,1,float16,fp8,16383,0.033530667424201965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1,0.009519999846816063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,1,0.012666666259368261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,3,0.012874666601419449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,7,0.012752000242471695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,3,0.009610666582981745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,7,0.009685333197315535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,15,0.01002133327225844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,31,0.009754666437705358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,15,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,31,0.012826666235923767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,63,0.009962666779756546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,63,0.012714666624863943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,127,0.010341333225369453
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,127,0.013253333667914072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,255,0.015386667102575302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,511,0.021189334491888683
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,511,0.01773333301146825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,255,0.01463466634353002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,1023,0.02370133250951767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1023,0.02629333237806956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1,0.013850666582584381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,2047,0.03618133316437403
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,1,128,1,float16,fp8,2047,0.03589866558710734
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,1,0.012863999853531519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,3,0.013936000565687815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,3,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,7,0.013957332819700241
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,15,0.014015999933083853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,7,0.01292266696691513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,31,0.012917333592971167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,63,0.014335999886194864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,63,0.013162666310866674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,15,0.012821332861979803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,31,0.014111999422311783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,127,0.01492799942692121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,127,0.013658666362365087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,255,0.023039999107519787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,255,0.015077333897352219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,511,0.028165332973003387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,511,0.018325333793958027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1,0.017488000293572743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1023,0.037946666280428566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,1023,0.02456533412138621
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1,0.015018666783968607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,3,0.017637333522240322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,2,128,1,float16,fp8,2047,0.037151999771595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,3,0.01481066644191742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,2047,0.20332799355189005
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,7,0.017903999735911686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,15,0.01783466711640358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,15,0.014896000425020853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,7,0.014933332800865173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,31,0.014933332800865173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,63,0.015605332950750986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,127,0.019440000255902607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,127,0.0164533331990242
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,255,0.0244159996509552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,31,0.01802666609485944
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,63,0.018229333062966663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,255,0.019071999937295914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,511,0.03429333368937174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,2047,0.3909333149592082
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,511,0.0244159996509552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,2047,0.19907732804616293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1023,0.2021333376566569
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,4,128,1,float16,fp8,1023,0.03480533262093862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1,0.02446399877468745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,7,0.02513066679239273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,3,0.025061334172884624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,7,0.021045332153638203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1,0.021013334393501282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,3,0.021045332153638203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,31,0.025333332518736523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,31,0.020960000654061634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,63,0.025941332181294758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,63,0.021381333470344543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,127,0.02288000037272771
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,15,0.02513599892457326
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,127,0.02775466690460841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,15,0.021018666525681812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,255,0.037050666908423104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,255,0.02794666588306427
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1023,0.3916906515757243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,2047,0.37786134084065753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,2047,0.7605760097503662
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,511,0.20348799228668213
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1,0.008687999720374743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,3,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,511,0.037685332198937736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,40,8,128,1,float16,fp8,1023,0.19833066066106161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,3,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,1,0.011055999745925268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,15,0.008826666822036108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,7,0.008965333302815756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,15,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,63,0.00897066667675972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,31,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,127,0.00922133338948091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,127,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,255,0.012565333396196365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,7,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,511,0.012954667210578918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1023,0.013642666240533194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,255,0.011973333855470022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,511,0.011920000116030375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,1023,0.012272000312805176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,2047,0.013749333719412485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,2047,0.01488000030318896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,8191,0.02271999915440877
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,4095,0.017909333109855652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,8191,0.027210667729377747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,4095,0.016656000167131424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,31,0.00903466654320558
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,1,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,16383,0.03262399882078171
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,3,0.011183999478816986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,3,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,1,128,1,float16,fp8,16383,0.034847999612490334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,7,0.012554666648308435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,7,0.011061333119869232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,15,0.012378666549921036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,15,0.011109333485364914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,31,0.01259200026591619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,63,0.012517333030700684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,31,0.011183999478816986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,127,0.012789333860079447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,63,0.011264000087976456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,127,0.01146666705608368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,255,0.012015999605258306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,511,0.015552000453074774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,255,0.01516266663869222
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1023,0.01621866722901662
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,511,0.01209066684047381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,1023,0.01221866657336553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,2047,0.01798933371901512
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,2047,0.013861333330472311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,4095,0.024666666984558105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,4095,0.016949333250522614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,8191,0.022810667753219604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,8191,0.03002133220434189
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,16383,0.03977599988381068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,2,128,1,float16,fp8,16383,0.035114665826161705
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1,0.011317333827416102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,3,0.012378666549921036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,7,0.012613333761692047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,3,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,15,0.01240533341964086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,7,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,31,0.012448000411192576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,63,0.012666666259368261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,15,0.011418666690587997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,31,0.011322667201360067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,63,0.011381333072980246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,127,0.012757333616415659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,127,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,255,0.015119999647140503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,511,0.01551466683546702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,255,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1023,0.016714667280515034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,511,0.012538666526476542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,2047,0.0223786657055219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,1023,0.012837332983811697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,2047,0.014368000129858652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,4095,0.027450665831565857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,4095,0.017456000049908955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,8191,0.03737066686153412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,8191,0.023898666103680927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,16383,0.20349333683649698
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,4,128,1,float16,fp8,16383,0.03606399893760681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,3,0.012453333785136541
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1,0.013104000439246496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,3,0.01321600005030632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,15,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,7,0.013232000172138214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,7,0.012629333883523941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,31,0.012613333761692047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,15,0.013173333058754602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,31,0.013183999806642532
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,63,0.012586666891972223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,63,0.013183999806642532
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,127,0.012789333860079447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,255,0.014368000129858652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,511,0.015925332903862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,127,0.013712000101804733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,255,0.01526933287580808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,511,0.014805333067973455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1023,0.021183999876181286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,2047,0.02621866762638092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,4095,0.04035199930270513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,1023,0.015573333948850632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,4095,0.02332799881696701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,2047,0.018165333817402523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,8191,0.033610666791598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,8191,0.2011786699295044
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,16383,0.38794668515523273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,40,8,128,1,float16,fp8,16383,0.19453867276509604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,1,0.01379199946920077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,3,0.013855999956528345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1,0.009839999799927076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,3,0.009904000287254652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,7,0.01003200002014637
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,7,0.013834666460752487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,15,0.010122666756312052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,15,0.013925333817799887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,31,0.013898666948080063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,31,0.010154666379094124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,63,0.01044800008336703
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,63,0.01422400027513504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,127,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,255,0.023018665611743927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,127,0.014896000425020853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,255,0.01613333324591319
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,511,0.027893332143624622
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,511,0.019120000302791595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1,0.01844800015290578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1023,0.037578667203585304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,1,128,1,float16,fp8,1023,0.025765334566434223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,1,0.016106666376193363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,3,0.018751999984184902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,3,0.016010666886965435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,15,0.018698666244745255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,7,0.018629333625237148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,31,0.016010666886965435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,7,0.01590399940808614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,15,0.015935999651749928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,31,0.019018666197856266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,63,0.01947733387351036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,63,0.016501333564519882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,127,0.02046400060256322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,127,0.01724799970785777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,255,0.025578667720158894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,255,0.019920000185569126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,511,0.035173334181308746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,511,0.025514667232831318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1,0.02550933261712392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1023,0.19945067167282104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,3,0.025637333591779072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,2,128,1,float16,fp8,1023,0.03537066777547201
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1,0.022106667359670002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,7,0.026149332523345947
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,3,0.021967999637126923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,7,0.02205866575241089
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,15,0.021930667261282604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,31,0.022277332842350006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,63,0.026917333404223125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,63,0.022389332453409832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,15,0.026181332767009735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,31,0.0262773334980011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,127,0.028575999041398365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,127,0.02399466683467229
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,255,0.038586666186650596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,255,0.029103999336560566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1,0.03924266745646795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1,0.033615998923778534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,511,0.20702399810155234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,3,0.04004266609748205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,511,0.038949333131313324
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,3,0.033615998923778534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,7,0.040463998913764954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,7,0.033546666304270424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,4,128,1,float16,fp8,1023,0.20182400941848755
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1023,0.39851200580596924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,15,0.03369066615899404
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,31,0.03369066615899404
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,63,0.041450666884581246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,15,0.04053866614898046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,63,0.033887999753157295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,31,0.041002665956815086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,255,0.20742400487263998
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1023,0.7730507055918375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,127,0.03478399912516276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,127,0.04389866689840952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,255,0.04498666524887085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,1023,0.38306132952372235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,511,0.4000053405761719
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,40,8,128,1,float16,fp8,511,0.20163732767105103
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,1,0.01806933308641116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,7,0.01825599993268649
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,1,0.014495999862750372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,3,0.01451733335852623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,3,0.018218666315078735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,7,0.014666666587193808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,31,0.014864000181357065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,31,0.01820266619324684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,15,0.018165333817402523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,63,0.01533866673707962
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,63,0.018383999665578205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,15,0.014671999961137772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,127,0.0164533331990242
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,127,0.01952533299724261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,255,0.02144533395767212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,1,0.02805333336194356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,1,128,1,float16,fp8,255,0.0222080002228419
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,1,0.02437866727511088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,3,0.02827200045188268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,7,0.02869333326816559
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,3,0.024143998821576435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,7,0.02441066751877467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,15,0.028650666276613872
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,15,0.02426133304834366
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,31,0.029098667204380035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,31,0.024314666787783306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,63,0.02940266579389572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,63,0.024832000335057575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,127,0.031317333380381264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,127,0.026629333694775898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,255,0.04134399940570196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,2,128,1,float16,fp8,255,0.03178666780392329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,1,0.04212800165017446
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,1,0.0359946663180987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,3,0.04275199770927429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,3,0.036176001032193504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,7,0.043375998735427856
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,7,0.03618666778008143
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,15,0.04333333174387614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,15,0.036090667049090065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,31,0.043712000052134194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,31,0.03587199995915095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,63,0.04458666841189066
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,63,0.036320000886917114
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,127,0.04691733419895172
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,127,0.037632000943024956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,255,0.21427200237909952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,1,0.06677866478761037
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,4,128,1,float16,fp8,255,0.04764799773693085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,3,0.06765333314736684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,1,0.05530133346716563
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,7,0.06883200009663899
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,3,0.055434669057528176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,7,0.05566399792830149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,15,0.05563200016816457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,31,0.06936533252398173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,31,0.055786664287249245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,63,0.07102400064468384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,15,0.06866666674613953
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,63,0.05634133517742157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,127,0.23518933852513632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,127,0.05890133480230967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,255,0.41120533148447674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,40,8,128,1,float16,fp8,255,0.2097760041554769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,1,0.02864533414443334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,1,0.023077333966890972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,3,0.02348266790310542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,3,0.028912000358104706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,7,0.02380799998839696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,15,0.0286613330245018
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,31,0.02401600033044815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,7,0.028751999139785767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,31,0.02869333326816559
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,63,0.024512000381946564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,15,0.023904000719388325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,63,0.02956799914439519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,127,0.026714667677879333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,1,0.04671466847260793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,1,0.04065066576004028
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,1,128,1,float16,fp8,127,0.03092266619205475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,3,0.04717333118120829
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,3,0.040821333726247154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,7,0.04799999793370565
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,7,0.04083733260631561
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,15,0.04049599915742874
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,31,0.04845866560935974
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,15,0.04816000163555145
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,31,0.0406986673672994
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,63,0.0491946687301
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,63,0.04154133299986521
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,127,0.05203733344872793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,2,128,1,float16,fp8,127,0.04262933135032654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,1,0.07222400108973186
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,1,0.06181333462397257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,3,0.07390933235486348
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,3,0.061674664417902626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,7,0.07437866429487865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,15,0.07482133309046428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,7,0.06191466748714447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,15,0.06225599845250448
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,31,0.07529066503047943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,31,0.06195199986298879
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,63,0.07712533573309581
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,63,0.06258133550484975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,127,0.23217066129048666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,1,0.1226026713848114
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,4,128,1,float16,fp8,127,0.064751997590065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,1,0.10136533776919048
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,3,0.12497066458066304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,3,0.10097066561381023
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,7,0.12710400422414145
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,7,0.10110400120417277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,15,0.1262399951616923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,15,0.10122666756312053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,31,0.10197333494822185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,31,0.12946666280428568
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,1,0.011168000598748526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,3,0.011183999478816986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1,0.008976000050703684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,7,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,15,0.009056000038981438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,63,0.23289066553115845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,63,0.10249066352844238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,15,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,31,0.009109333157539368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,63,0.009141333401203156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,3,0.00902399979531765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,127,0.009375999992092451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,7,0.009130666653315226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,255,0.012842666357755661
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,255,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,127,0.011322667201360067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,511,0.013295999417702356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,511,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,31,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,63,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,1023,0.013829333086808523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1023,0.014032000054915747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,2047,0.015989333391189575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,2047,0.016917333006858826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,8191,0.028565332293510437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,4095,0.022831998765468597
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,4095,0.023242667317390442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,8191,0.03474666674931844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,16383,0.037802666425704956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,127,0.43346667289733887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,1,128,1,float16,fp8,16383,0.05877333382765452
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,3,0.01118933285276095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,7,0.012757333616415659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,3,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,1,0.011194666226704916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,15,0.012773333738247553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,15,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,31,0.012762666990359625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,63,0.012847999731699625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,31,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,63,0.011354666203260422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,255,0.015354666858911514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,127,0.011488000551859537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,255,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,127,0.013066666821638743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,511,0.015962666521469753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,40,8,128,1,float16,fp8,127,0.2467893362045288
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,7,0.011391999820868174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,511,0.012576000144084295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,1023,0.014256000518798828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1023,0.017050666113694508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,2047,0.02271466702222824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,2047,0.01728533332546552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,4095,0.028101332485675812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,4095,0.02327466756105423
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,8191,0.03525333354870478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,16383,0.19966399669647217
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,3,0.012800000607967377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1,0.011498666057984034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,2,128,1,float16,fp8,16383,0.05889600018660227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,3,0.01145600030819575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,7,0.011472000430027643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,15,0.01267733300725619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1,0.012746666868527731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,7,0.012655999511480331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,31,0.012831999609867731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,15,0.011493333925803503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,31,0.011509332805871964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,63,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,63,0.011605333536863327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,127,0.011887999872366587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,127,0.013002666334311167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,8191,0.037920000652472176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,511,0.016384000579516094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,255,0.012703999876976013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,255,0.015541333705186844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,511,0.013157332936922709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,2047,0.026677332818508148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1023,0.02145066608985265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,1023,0.014682666709025701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,4095,0.04078399886687597
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,2047,0.017674667139848072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,8191,0.20358399550120035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,8191,0.03624533365170161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,4095,0.023887999355793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1,0.012608000387748083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,16383,0.391978661219279
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,4,128,1,float16,fp8,16383,0.19272534052530924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,3,0.012757333616415659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,3,0.013386666774749756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,15,0.012821332861979803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,7,0.012784000486135483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,7,0.013343999783198038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,31,0.012960000584522883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,15,0.013317332913478216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,63,0.013621332744757334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,127,0.013376000026861826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,31,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,255,0.016117333124081295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,127,0.01404800017674764
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,63,0.013045333325862885
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,255,0.014917333920796713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,2047,0.03562666724125544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1023,0.026234666506449383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,511,0.02091199904680252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,1023,0.01855466639002164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,511,0.015957333147525787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,2047,0.023754666248957317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,8191,0.38734400272369385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,8191,0.1946559945742289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,4095,0.033717334270477295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,4095,0.22175467014312744
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,40,8,128,1,float16,fp8,16383,0.37243199348449707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,16383,0.7562026977539062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,1,0.04012266546487808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,1,0.03365333378314972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,3,0.03381866713364919
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,3,0.040933333337306976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,7,0.04142399877309799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,15,0.041109333435694374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,7,0.033957332372665405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,15,0.034058667719364166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,31,0.04181333382924398
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,1,0.08170666793982188
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,31,0.03389866650104523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,1,0.05400000015894572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,1,128,1,float16,fp8,63,0.033914667864640556
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,3,0.08260266482830048
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,3,0.054234668612480164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,63,0.04281599819660187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,7,0.08329066634178162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,15,0.083514670530955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,7,0.054144000013669334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,15,0.05416533350944519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,31,0.08518399794896443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,63,0.10831466317176819
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,31,0.05467733244101206
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,2,128,1,float16,fp8,63,0.0554666668176651
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,1,0.1337386667728424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,1,0.09521599610646565
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,3,0.13609066605567932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,3,0.09515200058619182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,7,0.13797866304715475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,7,0.09565866986910503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,15,0.1386186679204305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,15,0.09587732950846355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,31,0.1539520025253296
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,31,0.09605333209037781
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,63,0.2682346701622009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,4,128,1,float16,fp8,63,0.09734400113423665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,1,0.2362826665242513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,1,0.17483200629552206
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,3,0.2405386765797933
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,3,0.17469867070515951
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,7,0.24496533473332724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,7,0.174453337987264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,15,0.2642400066057841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,15,0.17461333672205606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,31,0.296944002310435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,31,0.17857599258422852
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,63,0.4776853322982788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,40,8,128,1,float16,fp8,63,0.23532267411549887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,1,0.07149333258469899
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,1,0.057946667075157166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,3,0.07332799832026164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,3,0.05835199852784475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,7,0.07356800138950348
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,7,0.05845333139101664
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,15,0.0739519993464152
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,15,0.05846933523813883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,31,0.08085866769154866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,1,128,1,float16,fp8,31,0.05840000013510386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,1,0.16267200311024985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,1,0.09831466277440389
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,3,0.1657493313153585
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,3,0.09880533814430237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,7,0.182586669921875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,7,0.09916266798973083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,15,0.20399999618530273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,15,0.09884799520174663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,2,128,1,float16,fp8,31,0.09923199812571208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,1,0.2667626738548279
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,1,0.18405866622924805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,3,0.28787734111150104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,31,0.23604265848795572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,3,0.18433600664138794
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,7,0.3188213308652242
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,7,0.18397865692774454
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,15,0.3301066756248474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,15,0.1850186586380005
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,31,0.3754986524581909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,1,0.4970133304595947
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,4,128,1,float16,fp8,31,0.19482133785883585
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,1,0.34266666571299237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,3,0.5393173297246298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1,0.00892800030608972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,1,0.011407999942700068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,3,0.009136000027259191
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,3,0.011402666568756104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,3,0.34192001819610596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,7,0.009162666896979014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,7,0.011407999942700068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,15,0.01137599969903628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,31,0.009205333267649015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,15,0.009136000027259191
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,31,0.011359999577204386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,7,0.5640106598536173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,63,0.009328000247478485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,63,0.01138666644692421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,127,0.009461333354314169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,255,0.013007999708255133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,255,0.012901333471139273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,511,0.013637332866589228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,511,0.01431999976436297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1023,0.014901333798964819
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,1023,0.01739199956258138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,2047,0.021557333568731945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,7,0.3434986670811971
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,127,0.011690666278203329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,2047,0.0232640008131663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,4095,0.02658133457104365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,4095,0.03525333354870478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,1,128,1,float16,fp8,8191,0.05913066864013672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1,0.012778667112191519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,3,0.012858666479587555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,1,0.011440000186363855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,3,0.011541333049535751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,15,0.566981315612793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,7,0.012901333471139273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,15,0.012810666114091873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,7,0.011546666423479715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,8191,0.03678400069475174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,31,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,31,0.011472000430027643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,15,0.011445333560307821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,127,0.013237333546082178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,63,0.011701333026091257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,127,0.011855999628702799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,255,0.01571200042963028
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,255,0.013199999928474426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,511,0.016693333784739178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,511,0.014576000471909841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1023,0.022133332987626392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,1023,0.01748266691962878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,15,0.3485013246536255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,31,0.6196906566619873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,2047,0.02733866622050603
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,2047,0.02378133436044057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,63,0.01322666679819425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,4095,0.041135999063650765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,4095,0.03555200000603994
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1,0.011706666400035223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1,0.013002666334311167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,40,8,128,1,float16,fp8,31,0.3693866729736328
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,2,128,1,float16,fp8,8191,0.059674665331840515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,3,0.01163200040658315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,7,0.011786667009194693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,7,0.012896000097195307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,15,0.013093333691358566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,15,0.011626667032639185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,8191,0.19843733310699463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,31,0.01173866664369901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,127,0.01358933374285698
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,63,0.013151999562978745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,63,0.011792000383138657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,127,0.012181332955757776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,255,0.0163680004576842
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,255,0.013637332866589228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,511,0.015098666151364645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,511,0.021365332106749218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,3,0.012991999586423239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,2047,0.03597866743803024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,1023,0.018138666947682697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1023,0.026346666117509205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,2047,0.024538666009902954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,4095,0.036943999429543815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,4095,0.20260266462961832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,8191,0.39016000429789227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,31,0.013157332936922709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,3,0.01302933320403099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1,0.012906666845083237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,3,0.01349866638580958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,4,128,1,float16,fp8,8191,0.19388800859451294
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,15,0.01310933381319046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,7,0.013327999661366144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,7,0.013733333597580591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,31,0.01340266689658165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,15,0.013594667116800943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,127,0.013999999811251959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,63,0.013429333766301474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,31,0.013669333110253016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,63,0.013888000200192133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,127,0.01444799949725469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,255,0.016229332735141117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,511,0.025920001169045765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,255,0.021061333517233532
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1023,0.035749333600203194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,511,0.018944000204404194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,2047,0.03421866645415624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,1023,0.0244159996509552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,2047,0.22112532456715903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,4095,0.3875253200531006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,4095,0.19592533508936563
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,1,0.011776000261306763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,3,0.01184533288081487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,40,8,128,1,float16,fp8,8191,0.3731786807378133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,3,0.009216000015536943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,7,0.00922133338948091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,8191,0.7708960374196371
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,7,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,15,0.009189333145817121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,15,0.011781333635250727
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,31,0.011786667009194693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,63,0.009477333476146063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,31,0.009322666873534521
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,63,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,127,0.012053333222866058
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,127,0.009749333063761393
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,255,0.013525333255529404
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,255,0.01349866638580958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,511,0.014432000617186228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,511,0.016480000068744022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,1023,0.022543999056021374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1023,0.02021866664290428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,2047,0.025285333395004272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,8191,0.10597866773605347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,4095,0.038906666139761605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,8191,0.19937066237131754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,2047,0.034645333886146545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,1,128,1,float16,fp8,4095,0.05842666824658712
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1,0.012746666868527731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1,0.011909333368142446
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,3,0.012826666235923767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,3,0.011866666376590729
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,31,0.01320533330241839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,15,0.012928000340859095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,7,0.011978667229413986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,15,0.011802667131026586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,63,0.01313599944114685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,31,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,7,0.012773333738247553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,63,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,127,0.0136266661187013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,255,0.016410666207472484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,127,0.012362666428089142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,255,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,511,0.02142400046189626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,511,0.016794666647911072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1023,0.026746665438016255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,1023,0.023002666731675465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,4095,0.19847466548283896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,4095,0.05884799857934316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,2047,0.036415999134381614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,2047,0.03507733345031738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,8191,0.38118934631347656
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,2,128,1,float16,fp8,8191,0.19348265727361044
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1,0.013013333082199097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1,0.01211200033624967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,3,0.012991999586423239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,7,0.01191466674208641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,15,0.013194666554530462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,15,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,3,0.011968000481526056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,31,0.013141332815090815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,31,0.01190399999419848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,63,0.013376000026861826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,63,0.01231466606259346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,7,0.013194666554530462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,127,0.01402666668097178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,127,0.01259200026591619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,255,0.021114667256673176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,255,0.014240000396966934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,511,0.026165333886941273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,2047,0.20318933327992758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,511,0.017535999417304993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,2047,0.03612266729275385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1023,0.036159999668598175
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,1023,0.023792001108328503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,4095,0.3895519971847534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,4095,0.1934666633605957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1,0.016565332810084026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,4,128,1,float16,fp8,8191,0.36950401465098065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,8191,0.7571307023366293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,3,0.014064000298579534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,7,0.016917333006858826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1,0.01421333352724711
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,7,0.014127999544143677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,15,0.016864000509182613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,15,0.014245333770910898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,31,0.01699200024207433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,31,0.014111999422311783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,3,0.016821333517630894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,63,0.014826666563749313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,127,0.018538666268189747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,63,0.017781333376963932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,127,0.015696000307798386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,255,0.023408000667889912
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1023,0.19942933320999146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,1023,0.03366933266321818
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,2047,0.3863946596781413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,255,0.018266666680574417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,511,0.03306666761636734
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,2047,0.19576533635457358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,511,0.02362666775782903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,4095,0.37269333998362225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,3,0.008394666636983553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1,0.008565333361426989
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,15,0.008570666735370954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,7,0.00877333308259646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,7,0.010559999694426855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,32,8,128,1,float16,fp8,8191,0.727013349533081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,1,0.01055466632048289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,3,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,4095,0.752784013748169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,15,0.010501333822806677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,63,0.010661333799362183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,127,0.009066666786869368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,31,0.008767999708652496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,31,0.010645333677530289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,8191,1.657973289489746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,63,0.008810666700204214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,255,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,127,0.010938666760921478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,255,0.011519999553759893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,511,0.012741333494583765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1023,0.013162666310866674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,511,0.011584000041087469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,1023,0.011551999797423681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,2047,0.011626667032639185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,4095,0.012063999970753988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,4095,0.01669866715868314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,2047,0.014384000251690546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,8191,0.021194666624069214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,8191,0.013546666751305262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,32767,0.03365333378314972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,16383,0.03065066784620285
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,16383,0.016794666647911072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,1,128,1,float16,fp8,32767,0.022682666778564453
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,3,0.0086666668454806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1,0.008602666358153025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,7,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,3,0.010682666053374609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,15,0.008672000219424566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,7,0.010650667051474253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,31,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,15,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,63,0.00871999996403853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,63,0.010703999549150467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,31,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,127,0.008976000050703684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,255,0.013258667041858038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,255,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,127,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,511,0.012506666282812754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,511,0.011519999553759893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1023,0.013210666676362356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,1023,0.011663999408483505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,2047,0.01444799949725469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,4095,0.016805333395799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,2047,0.012058666596810022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,4095,0.0124746672809124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,8191,0.02161066730817159
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,16383,0.03512533257404963
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,8191,0.013914667069911957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,16383,0.01695466662446658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,32767,0.04041066765785217
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1,0.008672000219424566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,2,128,1,float16,fp8,32767,0.023029332359631855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,3,0.008682666967312494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,3,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,7,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,15,0.008757333581646284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,7,0.010714666297038397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,15,0.010794666906197866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,31,0.008757333581646284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,63,0.00897066667675972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,31,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,63,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,127,0.009093333035707474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,255,0.012282667060693106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,127,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,255,0.011584000041087469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,511,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,511,0.011802667131026586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1023,0.013295999417702356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,2047,0.014501333236694336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,1023,0.011909333368142446
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,4095,0.0173333336909612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,2047,0.012223999947309494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,8191,0.026213333010673523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,4095,0.012821332861979803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,8191,0.014592000593741735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,16383,0.03133333226044973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,32767,0.04090133309364319
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,16383,0.017642666896184284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1,0.008650666723648706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,4,128,1,float16,fp8,32767,0.024058667321999867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,3,0.008816000074148178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1,0.011152000476916632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,7,0.008703999842206636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,15,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,3,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,7,0.01109333336353302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,31,0.008933333059151968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,63,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,31,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,63,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,255,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,127,0.009061333412925402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,127,0.011370666325092316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,511,0.012655999511480331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,255,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1023,0.01331199953953425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,4095,0.02179199953873952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,1023,0.01421333352724711
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,2047,0.015008000036080679
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,511,0.013898666948080063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,2047,0.01469333345691363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,8191,0.026762666801611584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,4095,0.015669333438078564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,16383,0.03661333272854487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,8191,0.01830400029818217
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1,0.008639999975760778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,32767,0.03436266630887985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,1,0.010640000303586325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,32,8,128,1,float16,fp8,16383,0.023605334262053173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,32767,0.22136000792185465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,3,0.00877333308259646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,3,0.010858666151762009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,15,0.008693333094318708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,7,0.010890666395425797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,7,0.008687999720374743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,15,0.010634666929642359
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,31,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,31,0.008863999818762144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,63,0.010672000547250112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,63,0.009008000294367472
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,127,0.009279999881982803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,255,0.012346666306257248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,127,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,255,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,511,0.012746666868527731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,511,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,1023,0.011765333513418833
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,2047,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,2047,0.014495999862750372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1023,0.013424000392357508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,8191,0.02195200075705846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,4095,0.017055999487638474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,16383,0.036271999279658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,4095,0.01351999988158544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,8191,0.016688000410795212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,16383,0.022431999444961548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,32767,0.04121600091457367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,1,128,1,float16,fp8,32767,0.03459733227888743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1,0.012074666718641916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,7,0.012170666207869848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,7,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,15,0.01209066684047381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,3,0.012074666718641916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,15,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,31,0.012159999459981918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,63,0.012234666695197424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,31,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,127,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,255,0.014837333311637243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,255,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,127,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,511,0.015189333508412043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,511,0.01180800050497055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1023,0.01578666642308235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,1023,0.012037333101034164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,2047,0.012191999703645706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,4095,0.019786667078733444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,4095,0.013728000223636627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,2047,0.017071999609470367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,8191,0.029215998947620392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,8191,0.016688000410795212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,16383,0.034117333590984344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,16383,0.022709332406520844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,32767,0.04398933549722036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,2,128,1,float16,fp8,32767,0.035061334570248924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,3,0.01209066684047381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1,0.010954666882753372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,7,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,3,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,15,0.012122667084137598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,31,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1,0.012138667205969492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,15,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,63,0.012437333663304647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,31,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,63,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,255,0.014906667172908783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,127,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,255,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,511,0.015189333508412043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,127,0.012549333274364471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,511,0.01192533348997434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1023,0.01595199977358182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,2047,0.01773333301146825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,1023,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,2047,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,4095,0.02455466737349828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,8191,0.029498666524887085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,8191,0.01743999992807706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,16383,0.039333333571751915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,4095,0.01431999976436297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,16383,0.02362666775782903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1,0.012144000579913458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,32767,0.20573333899180093
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,4,128,1,float16,fp8,32767,0.03643733263015747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1,0.011706666400035223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,3,0.01221866657336553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,15,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,7,0.011551999797423681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,3,0.011157333850860596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,31,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,7,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,15,0.012159999459981918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,127,0.012533333152532578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,31,0.011722666521867117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,255,0.014959999670584997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,63,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,63,0.011690666278203329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,511,0.015333333363135656
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,127,0.011317333827416102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,255,0.013994666437307993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,511,0.01404800017674764
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1023,0.01657066618402799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,1023,0.014426667243242264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,2047,0.02216000109910965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,8191,0.03678400069475174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,2047,0.01544533297419548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,4095,0.018250666558742523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,4095,0.027280000348885853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,16383,0.20210667451222739
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,8191,0.023472001155217487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,16383,0.03342933456103007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,32767,0.38739200433095294
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,32,8,128,1,float16,fp8,32767,0.19404800732930502
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,1,0.012373333175977072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1,0.00956266683836778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,3,0.012602667013804117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,3,0.009493333597977957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,7,0.009557333464423815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,7,0.012373333175977072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,15,0.009573333586255709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,15,0.012448000411192576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,31,0.012602667013804117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,31,0.009797333429257074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,63,0.009829333052039146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,63,0.012639999389648438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,127,0.010202666744589806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,127,0.012778667112191519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,255,0.014501333236694336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,255,0.014965333044528961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,511,0.017456000049908955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,511,0.020432000358899433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1023,0.02553066611289978
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,1023,0.023445333043734234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,2047,0.03513599932193756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1,0.01368533323208491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,2047,0.0356480007370313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1,0.012608000387748083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,3,0.013562666873137156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,1,128,1,float16,fp8,4095,0.05931733548641205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,7,0.013712000101804733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,3,0.012613333761692047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,15,0.013674666484196981
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,7,0.01267733300725619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,4095,0.20083200931549072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,15,0.012554666648308435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,31,0.012736000120639801
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,63,0.013946666071812311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,31,0.013669333110253016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,63,0.012896000097195307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,127,0.014815999815861383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,255,0.022272000710169475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,511,0.027461332579453785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,511,0.018191999445358913
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,255,0.015087999403476715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,127,0.013301332791646322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1023,0.037151999771595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,1023,0.02420266717672348
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,4095,0.1949546734491984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,4095,0.38326934973398846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1,0.01462399959564209
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,2047,0.2013439933458964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1,0.017231999586025875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,2,128,1,float16,fp8,2047,0.03653866549332937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,3,0.017173333714405697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,3,0.014757333944241205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,7,0.017525333911180496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,7,0.014783999572197596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,15,0.017488000293572743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,31,0.017616000026464462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,63,0.015274666249752045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,63,0.01785600061217944
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,127,0.019098666807015736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,127,0.016063999384641647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,15,0.01479999969402949
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,31,0.014720000326633453
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,255,0.018922666708628338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,255,0.02405333270629247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,2047,0.39022934436798096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,511,0.03363200028737386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,511,0.0240639994541804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,2047,0.19792532920837402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,4095,0.760106643040975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,1023,0.034346667428811394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1,0.024330665667851765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1023,0.20185067256291708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,4,128,1,float16,fp8,4095,0.37564265727996826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,3,0.020773333807786305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1,0.0206986665725708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,15,0.024879999458789825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,7,0.024864000578721363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,31,0.025008000433444977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,3,0.024400000770886738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,31,0.020784000555674236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,63,0.025472000241279602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,63,0.021205333371957142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,7,0.020853333175182343
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,15,0.020810666183630627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,127,0.02720533311367035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,127,0.022485333184401195
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,255,0.02736533433198929
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,255,0.03673599908749262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1023,0.4282240072886149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,511,0.03749866783618927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,1023,0.19646400213241577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,511,0.20280534029006958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,2047,0.7582506338755289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,2047,0.3768426577250163
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,1,0.011061333119869232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,3,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,7,0.008746666833758354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,7,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,32,8,128,1,float16,fp8,4095,0.7327786286671957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,15,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,15,0.011087999989589056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,31,0.008922666932145754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1,0.008698666468262672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,31,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,3,0.008762666955590248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,63,0.008986666798591614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,63,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,127,0.009232000137368837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,127,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,255,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,4095,1.4962560335795085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,255,0.012517333030700684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,511,0.011871999750534693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,511,0.012768000364303589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1023,0.013503999759753546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,1023,0.012122667084137598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,2047,0.014762666076421738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,2047,0.013530666629473368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,4095,0.017759999881188076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,8191,0.026858667532602947
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,16383,0.032602667808532715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,4095,0.016629333297411602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,8191,0.022687998910744984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,16383,0.03453866640726725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1,0.012005332857370377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,32767,0.04197333256403605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,3,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,3,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,7,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,1,128,1,float16,fp8,32767,0.058559998869895935
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,7,0.011066666493813196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,31,0.012181332955757776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,63,0.012362666428089142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,31,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,15,0.012058666596810022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,63,0.011231999844312668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,127,0.012666666259368261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,127,0.011381333072980246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,511,0.015237333873907724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,255,0.014901333798964819
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,255,0.012026666353146235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1023,0.016085332880417507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,511,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,1023,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,2047,0.017535999417304993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,4095,0.024586667617162068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,4095,0.016965333372354507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,2047,0.01381333296497663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,8191,0.029861333469549816
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,8191,0.022976001103719074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,16383,0.03961600114901861
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,16383,0.034832000732421875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,3,0.012223999947309494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,2,128,1,float16,fp8,32767,0.05898666878541311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1,0.012202666451533636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1,0.011125333607196808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,32767,0.20015466213226318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,3,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,7,0.012282667060693106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,15,0.012202666451533636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,31,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,7,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,31,0.011120000233252844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,63,0.01246400053302447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,63,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,127,0.012522666404644648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,15,0.011317333827416102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,255,0.015002666662136713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,127,0.01146666705608368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,255,0.012133333832025528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,511,0.015504000087579092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,511,0.012618667135636011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1023,0.016501333564519882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,1023,0.012736000120639801
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,2047,0.014384000251690546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,4095,0.027434666951497395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,4095,0.01743999992807706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,8191,0.03711466739575068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,2047,0.02218666672706604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,8191,0.02351466566324234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,16383,0.2024959921836853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1,0.012154666086037954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,16383,0.035936000446478523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,4,128,1,float16,fp8,32767,0.1919040083885193
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,32767,0.39019731680552167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1,0.013045333325862885
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,7,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,3,0.01321600005030632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,15,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,3,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,7,0.013066666821638743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,15,0.013151999562978745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,31,0.01314666618903478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,31,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,63,0.013274667163689932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,255,0.015109332899252573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,127,0.012576000144084295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,63,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,255,0.014416000495354334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,127,0.013584000368913015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1023,0.021007999777793884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,511,0.01594666639963786
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,511,0.014815999815861383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,1023,0.015541333705186844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,2047,0.026170666019121807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,2047,0.018239999810854595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,4095,0.040048000713189445
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,4095,0.023423999547958374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,8191,0.03385066737731298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,8191,0.200981338818868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,16383,0.3872799873352051
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,16383,0.19405867656071982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,32767,0.754197359085083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,32,8,128,1,float16,fp8,32767,0.39876266320546466
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1,0.009754666437705358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,1,0.013573333621025085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,3,0.013429333766301474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,3,0.009904000287254652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,7,0.009904000287254652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,7,0.013386666774749756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,15,0.009919999788204828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,15,0.013562666873137156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,31,0.010106666634480158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,31,0.013866666704416275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,63,0.010437333335479101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,63,0.013850666582584381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,127,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,127,0.014256000518798828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,255,0.020997333029905956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,255,0.015930666277805965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,511,0.026261332134405773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,511,0.01884799947341283
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1023,0.03608533243338267
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1,0.018090666582187016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,1023,0.02499199906984965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1,0.015647999942302704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,2047,0.22155733903249106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,3,0.017984000345071156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,3,0.015749332805474598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,7,0.01841066653529803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,7,0.015610666324694952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,1,128,1,float16,fp8,2047,0.03807466725508372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,15,0.015717333803574245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,15,0.018272000054518383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,31,0.015578666081031164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,63,0.018944000204404194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,63,0.01624533285697301
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,127,0.020143999407688778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,31,0.018394666413466137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,255,0.02514133354028066
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,255,0.019498666127522785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,511,0.03486400097608566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,511,0.025045332809289295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,127,0.016976000120242436
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1,0.024986666937669117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1023,0.19844265778859457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1,0.021498667697111767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,1023,0.035232000052928925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,3,0.021477334201335907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,7,0.025594666600227356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,3,0.025605333348115284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,15,0.02565866708755493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,2,128,1,float16,fp8,2047,0.20099733273188272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,2047,0.3800853490829468
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,31,0.02163200080394745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,31,0.025813333690166473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,63,0.02651199946800868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,7,0.02170666555563609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,15,0.021546666820844013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,63,0.021882665654023487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,255,0.028245332340399425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,255,0.03811733424663544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,511,0.20617065827051798
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,127,0.028170667588710785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,511,0.03861333429813385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,127,0.023455999791622162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,2047,0.38115731875101727
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,2047,0.7659146785736084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1,0.03317866722742716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1,0.03902933249870936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,3,0.03298133363326391
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,4,128,1,float16,fp8,1023,0.20055999358495077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,3,0.03937066594759623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,7,0.03294399877389272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1023,0.43187201023101807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,7,0.039877332746982574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,15,0.0331839993596077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,15,0.04035733391841253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,31,0.04044266790151596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,31,0.03329599897066752
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,63,0.04121600091457367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,127,0.043194666504859924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,127,0.03427733232577642
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,63,0.0332640012105306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,255,0.04442666471004486
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,255,0.2069759964942932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1023,0.769536018371582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,1023,0.38150401910146076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,511,0.3971253236134847
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,511,0.200272003809611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,32,8,128,1,float16,fp8,2047,0.7357439994812012
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,2047,1.5105172793070476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,1,0.017152000218629837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,1,0.01392000044385592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,3,0.014181333283583323
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,7,0.017093333105246227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,15,0.014149333039919535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,31,0.017370666066805523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,31,0.01451733335852623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,15,0.017210666090250015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,63,0.014874666929244995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,7,0.014085333794355392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,3,0.01722666621208191
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,63,0.017701332767804463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,127,0.01609066625436147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,127,0.018751999984184902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,255,0.021162666380405426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,255,0.021312000850836437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,1,0.027087998886903126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,511,0.031018666923046112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,1,128,1,float16,fp8,511,0.026330667237440746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,1,0.023221333821614582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,3,0.02741333345572154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,3,0.023397333920001984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,7,0.027621333797772724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,7,0.023232000569502514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,15,0.027658666173617046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,15,0.023530667026837666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,31,0.02790933350721995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,31,0.023386667172114056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,63,0.023984000086784363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,127,0.025445332129796345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,127,0.030218665798505146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,255,0.04031999905904134
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,1,0.04131733377774557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,255,0.030591999491055805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,511,0.20409599939982095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,3,0.041759997606277466
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,2,128,1,float16,fp8,511,0.04011200120051702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,3,0.03484266748030981
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,1,0.03465066601832708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,7,0.04236799975236257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,7,0.03483733286460241
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,15,0.03482666611671448
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,31,0.03496533383925756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,63,0.043525333205858864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,15,0.042037333051363625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,31,0.042693331837654114
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,63,0.03523733218510946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,127,0.045994664231936135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,127,0.036133334040641785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,255,0.21228265762329102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,255,0.04624533156553904
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,1,0.06589333216349284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,511,0.40677332878112793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,4,128,1,float16,fp8,511,0.20749332507451376
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,3,0.06657066444555919
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,1,0.05444266895453135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,7,0.054671997825304665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,3,0.05434666574001312
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,7,0.0675786683956782
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,31,0.06832533578077953
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,31,0.05497066676616669
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,15,0.06735466420650482
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,63,0.07000533243020375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,63,0.05515733361244202
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,15,0.05426133175690969
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,127,0.2127093275388082
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,127,0.05704533557097117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,255,0.4432533184687297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,1,0.022431999444961548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,255,0.20664000511169434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,1,0.027029333015282948
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,511,0.78056534131368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,32,8,128,1,float16,fp8,511,0.3893066644668579
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,3,0.022672000030676525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,7,0.02276800076166789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,3,0.0271519993742307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,15,0.023029332359631855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,7,0.02693866689999898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,31,0.027237333357334137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,63,0.02378133436044057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,63,0.02777066578467687
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,127,0.025722667574882507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,15,0.027141332626342773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,31,0.023290666441122692
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,127,0.029146666328112285
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,1,128,1,float16,fp8,255,0.03457599878311157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,1,0.039093332986036934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,255,0.035674666364987694
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,3,0.045423999428749084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,1,0.04484800000985464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,3,0.03871466716130575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,7,0.046015997727712
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,15,0.04633066554864248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,31,0.0390133336186409
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,31,0.04651199777921041
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,7,0.03907199949026108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,15,0.038917332887649536
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,63,0.04729066789150238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,63,0.039690665900707245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,127,0.050053333242734276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,127,0.04038399954636892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,1,0.07041599849859874
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,255,0.21826666593551636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,1,0.059530665477116905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,2,128,1,float16,fp8,255,0.051642666260401406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,3,0.0717493345340093
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,3,0.05985066791375478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,7,0.07262933254241943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,7,0.059706668059031166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,15,0.059765333930651345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,15,0.07272533575693767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,31,0.07344000041484833
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,31,0.06012799839178721
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,63,0.07503466804822286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,63,0.06057600180308024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,127,0.22453333934148154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,127,0.06260266900062561
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,255,0.4251146713892619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,4,128,1,float16,fp8,255,0.22075200080871582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,1,0.12095466256141663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,1,0.0995306670665741
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,3,0.12308266758918762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,3,0.0990613301595052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,7,0.12424000104268391
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,15,0.12499200304349263
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,15,0.0995093286037445
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,31,0.12761599818865457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,7,0.09961600104967754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,31,0.0997226635615031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1,0.00892800030608972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,1,0.011221333096424738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,63,0.10048000017801921
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,63,0.24612265825271606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,3,0.008986666798591614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,3,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,15,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,15,0.011157333850860596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,31,0.00898133342464765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,31,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,7,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,7,0.009008000294367472
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,63,0.009119999905427298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,63,0.011285333583752314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,127,0.009242666885256767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,127,0.011402666568756104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,255,0.012586666891972223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,255,0.012005332857370377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,511,0.012847999731699625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1023,0.013962666193644205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,511,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,1023,0.013663999736309052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,2047,0.015743999431530636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,4095,0.022874665757020313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,127,0.22315732638041177
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,127,0.4253653287887573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,2047,0.016832000265518825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,4095,0.02271999915440877
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,8191,0.028016000986099243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,16383,0.05857066810131073
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,32767,0.10626666744550069
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,32767,0.19961599508921304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1,0.01246400053302447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,3,0.012453333785136541
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,7,0.012597333639860153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,1,128,1,float16,fp8,8191,0.03475199888149897
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,7,0.011215999722480774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,3,0.011215999722480774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,15,0.012613333761692047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,15,0.011285333583752314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,31,0.012597333639860153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,31,0.011237333218256632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,63,0.012639999389648438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,255,0.8040693600972494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,63,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,127,0.012917333592971167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,127,0.011445333560307821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,255,0.015072000523408255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,16383,0.03809600075085958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,511,0.015754666179418564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,32,8,128,1,float16,fp8,255,0.4312906662623088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,511,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1023,0.016943999876578648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,1023,0.013893333574136099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,2047,0.022650666534900665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,2047,0.016794666647911072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,4095,0.022997332115968067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,8191,0.037578667203585304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,8191,0.03507733345031738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,16383,0.05884266893068949
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,16383,0.1980959971745809
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,32767,0.3810826539993286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,32767,0.19330666462580362
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1,0.012597333639860153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,3,0.012410666793584824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,3,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,7,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,15,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,7,0.011349332829316458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,15,0.01128000020980835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,31,0.012618667135636011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,31,0.011285333583752314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,63,0.012810666114091873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,63,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,127,0.012960000584522883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,127,0.0116799995303154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,255,0.015322666615247726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,255,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,511,0.01616000011563301
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,511,0.012703999876976013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1023,0.02142400046189626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,2047,0.026549334327379864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,1023,0.014416000495354334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,2047,0.017535999417304993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,4095,0.040565334260463715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,4095,0.023706667125225067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,8191,0.03617066641648611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,8191,0.2029013236363729
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,2,128,1,float16,fp8,255,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,4095,0.0276853342851003
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,16383,0.19196800390879312
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,16383,0.38973867893218994
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,4,128,1,float16,fp8,32767,0.36929599444071454
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,3,0.012634667257467905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,3,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,32767,0.7576800187428793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1,0.012554666648308435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,7,0.012879999975363413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,7,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,15,0.013258667041858038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,15,0.012752000242471695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,31,0.012869333227475485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,31,0.013477332890033722
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,63,0.01303999995191892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,127,0.01333333303531011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,63,0.01351999988158544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,127,0.013877333452304205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,255,0.014842666685581207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1023,0.025850666066010792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,511,0.015685333559910457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,511,0.020784000555674236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,255,0.01597333326935768
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,1023,0.018351999421914417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,2047,0.035530666510264076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,4095,0.2006346583366394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,2047,0.023669332265853882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,4095,0.03401066611210505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,8191,0.19382399320602417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,16383,0.3721546729405721
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,16383,0.7546559969584147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,8191,0.42793067296346027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,32,8,128,1,float16,fp8,32767,0.7269439697265625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,32767,1.4876319567362468
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,1,0.032773333291212715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,1,0.039103999733924866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,3,0.03950933367013931
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,3,0.033002667129039764
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,7,0.04009599983692169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,7,0.03297599901755651
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,15,0.039994666973749794
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,15,0.03295466552178065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,31,0.04065600037574768
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,31,0.03295466552178065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,63,0.04133866727352142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,63,0.03322133421897888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,127,0.0448586642742157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,1,0.0775733341773351
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,1,128,1,float16,fp8,127,0.03410666684309641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,3,0.07868266602357228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,1,0.05332799752553304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,3,0.05328533550103506
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,7,0.07994666695594788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,7,0.05345066885153452
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,15,0.07969599962234497
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,15,0.053344001372655235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,31,0.08075200021266937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,31,0.05380799869696299
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,63,0.09077866872151692
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,127,0.2476053237915039
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,63,0.054560000697771706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,1,0.12967999776204428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,1,0.09410132964452107
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,2,128,1,float16,fp8,127,0.05705599983533224
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,3,0.13222400347391763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,7,0.1341333289941152
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,3,0.09454400340716045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,7,0.09477866689364116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,15,0.13351999719937643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,15,0.0946666697661082
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,31,0.1423679987589518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,31,0.09503466884295146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,63,0.09629333019256592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,63,0.255295991897583
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,1,0.23218133052190146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,4,128,1,float16,fp8,127,0.22595733404159546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,1,0.17316800355911255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,127,0.4963200092315674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,3,0.2367146611213684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,7,0.24066666762034097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,3,0.17356266578038534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,7,0.17325333754221597
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,15,0.17362133661905924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,15,0.2503359913825989
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,31,0.2861599922180176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,31,0.17525333166122437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,63,0.46061333020528156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,63,0.2456000049908956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,127,0.8447893460591634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,32,8,128,1,float16,fp8,127,0.4508426586786906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,1,0.06929066777229309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,1,0.05675200124581655
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,3,0.07016000151634216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,3,0.05668266614278158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,7,0.07115200161933899
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,7,0.056613331039746605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,15,0.07075733443101247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,15,0.05667200187842051
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,31,0.07497600217660268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,31,0.05701333284378052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,63,0.14433067043622336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,1,128,1,float16,fp8,63,0.05764799813429514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,1,0.14432533582051596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,1,0.09682666261990865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,3,0.1479039986928304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,3,0.09739200274149577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,7,0.09703466296195984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,15,0.16788800557454428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,15,0.09745599826176961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,31,0.19981332619984946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,31,0.0976746678352356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,63,0.3116053342819214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,1,0.25086400906244916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,2,128,1,float16,fp8,63,0.10312533378601074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,1,0.18200532595316568
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,7,0.15226133664449057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,3,0.25998934110005695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,3,0.1818986733754476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,7,0.27979199091593426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,7,0.18210132916768393
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,15,0.30294400453567505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,15,0.18294399976730347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,31,0.34430400530497235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,63,0.5309120019276937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,63,0.2650559941927592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,1,0.458624005317688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,1,0.3394399881362915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,3,0.4957706530888875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,4,128,1,float16,fp8,31,0.18825600544611612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,3,0.3401973247528076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,7,0.5295946598052979
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1,0.009279999881982803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,1,0.011424000064531961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,3,0.009103999783595404
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,3,0.011440000186363855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,7,0.00926399976015091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,7,0.01129066695769628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,15,0.009130666653315226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,15,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,31,0.00926399976015091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,31,0.011472000430027643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,63,0.00927466650803884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,7,0.3407680193583171
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,15,0.5366453329722086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,63,0.011424000064531961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,127,0.011519999553759893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,255,0.012778667112191519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,511,0.013493333011865616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,511,0.014245333770910898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,127,0.00960533320903778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1023,0.014842666685581207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,255,0.013114667187134424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,1023,0.017258666455745697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,2047,0.02107733239730199
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,2047,0.02310933421055476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,4095,0.026176000634829204
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,4095,0.03525333354870478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,8191,0.03578133384386698
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,8191,0.059024001161257424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,16383,0.1986186703046163
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,31,0.5821546713511149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,1,128,1,float16,fp8,16383,0.10672000050544739
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1,0.012741333494583765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,3,0.012837332983811697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,7,0.01146666705608368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,7,0.012944000462690989
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,15,0.012858666479587555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,15,0.01145600030819575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,31,0.012975999464591345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,15,0.3447519938151042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,31,0.011488000551859537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,63,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,63,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,127,0.013381333400805792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,3,0.01145600030819575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,255,0.015605332950750986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,255,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,31,0.35902400811513263
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,511,0.016565332810084026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,511,0.014576000471909841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1023,0.021984001000722248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,1023,0.017397332936525345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,4095,0.04120533416668574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,2047,0.02718399961789449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,4095,0.03542399903138479
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,8191,0.19833600521087646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,127,0.01173866664369901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,8191,0.059530665477116905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,63,0.9983733495076498
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,16383,0.3829066753387451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1,0.012831999609867731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,16383,0.19405867656071982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,3,0.012837332983811697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,7,0.012991999586423239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,7,0.011578666667143503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,3,0.011637333780527115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,32,8,128,1,float16,fp8,63,0.47235198815663654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,31,0.012938667088747025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,15,0.011600000162919363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,63,0.01321600005030632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,31,0.011584000041087469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,2,128,1,float16,fp8,2047,0.02363733450571696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,63,0.011663999408483505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,127,0.013552000125249227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,127,0.01210133358836174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,255,0.016250666230916977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,255,0.013338666409254074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,511,0.014864000181357065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,2047,0.02430933217207591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,2047,0.03619733452796936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,4095,0.20273067553838095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,1023,0.01788266624013583
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1023,0.02621866762638092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,4095,0.0366293340921402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,8191,0.19386667013168335
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,8191,0.38917867342631024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,16383,0.7563253243764242
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1,0.013461332768201828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,3,0.013530666629473368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,3,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,4,128,1,float16,fp8,16383,0.36854398250579834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1,0.012981332838535309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,7,0.013178666432698568
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,15,0.013050666699806849
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,7,0.013562666873137156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,15,0.013552000125249227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,31,0.013199999928474426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,31,0.013536000003417334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,63,0.01331199953953425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,63,0.013818666338920593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,127,0.014554666976133982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,127,0.014090667168299357
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,15,0.013066666821638743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,255,0.016074666132529575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,255,0.020938667158285778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1023,0.035717333356539406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,511,0.02603733291228612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,2047,0.20164267222086588
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,511,0.018874666343132656
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,1023,0.024031999210516613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,2047,0.03409600009520849
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,4095,0.19497066736221313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,4095,0.38796265920003253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,511,0.021477334201335907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1,0.009039999917149544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,8191,0.3734453519185384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,1,0.011551999797423681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,8191,0.755413293838501
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,32,8,128,1,float16,fp8,16383,0.7255733013153076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,3,0.009045333291093508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,3,0.0116799995303154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,7,0.009152000149091085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,7,0.011535999675591787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,16383,1.4902453422546387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,15,0.009178666397929192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,15,0.011600000162919363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,31,0.00921066664159298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,31,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,63,0.009359999870260557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,63,0.011589333415031433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,127,0.01180800050497055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,127,0.00955200009047985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,255,0.013237333546082178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,255,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,511,0.016362667083740234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,511,0.014165333161751429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1023,0.019744000087181728
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,2047,0.03433600068092346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,1023,0.02221333235502243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,8191,0.19709867238998413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,4095,0.0582239975531896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,1,128,1,float16,fp8,8191,0.10586133599281311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,2047,0.02459733436505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,4095,0.03862400104602178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1,0.011530666301647821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1,0.012847999731699625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,3,0.012896000097195307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,7,0.012970666090647379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,15,0.011658667276302973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,31,0.013007999708255133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,31,0.011637333780527115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,7,0.011733333269755045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,63,0.01309866706530253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,3,0.011813333878914515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,15,0.012954667210578918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,63,0.011733333269755045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,127,0.013546666751305262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,127,0.012074666718641916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,255,0.016255999604860943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,255,0.01357866699496905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,511,0.021354667842388153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,511,0.016688000410795212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1023,0.026394667724768322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,1023,0.022826666633288067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,4095,0.19774933656056723
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,4095,0.05871466795603434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,2047,0.036277333895365395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,2047,0.03475199888149897
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,8191,0.379962682723999
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1,0.01302933320403099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1,0.011786667009194693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,2,128,1,float16,fp8,8191,0.19260267416636148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,3,0.01314666618903478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,7,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,15,0.013066666821638743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,7,0.013162666310866674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,15,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,31,0.013157332936922709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,3,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,31,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,63,0.012144000579913458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,127,0.013936000565687815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,63,0.013482666263977686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,127,0.012517333030700684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,255,0.014240000396966934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,255,0.020938667158285778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,511,0.02605333427588145
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,511,0.01727466657757759
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,2047,0.20188800493876138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,2047,0.036015999813874565
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1023,0.035599999129772186
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,1023,0.02367999901374181
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,4095,0.3877973159154256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,4095,0.1922880013783773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1,0.01393066719174385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,8191,0.7547946770985922
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1,0.01657066618402799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,4,128,1,float16,fp8,8191,0.3689813216527303
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,7,0.016714667280515034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,7,0.013818666338920593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,3,0.013978666315476099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,15,0.016965333372354507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,15,0.013898666948080063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,3,0.01674666628241539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,31,0.017103999853134155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,63,0.017263999829689663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,31,0.014005333185195923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,63,0.01441066712141037
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,127,0.018464000274737675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,127,0.015504000087579092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,255,0.02332799881696701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,255,0.018181333939234417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,511,0.03299200038115183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,1023,0.033285332222779594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1023,0.21874666213989258
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,511,0.02325333406527837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,2047,0.38548799355824787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,2047,0.1951733430226644
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,4095,0.7521066665649414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,3,0.008597333605090777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,3,0.01071999967098236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1,0.008602666358153025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,7,0.010698666175206503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,15,0.008714666590094566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,7,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,4095,0.3723306655883789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,15,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,24,8,128,1,float16,fp8,8191,0.7821013132731119
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,1,0.010682666053374609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,31,0.008687999720374743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,63,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,31,0.010714666297038397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,63,0.010666667173306147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,127,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,255,0.012272000312805176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,511,0.012661332885424295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,127,0.008896000062425932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,511,0.011488000551859537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,255,0.011616000284751257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1023,0.013077333569526672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,8191,1.490330696105957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,1023,0.011792000383138657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,2047,0.014261333892742792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,4095,0.016682667036851246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,4095,0.012117333710193634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,8191,0.021157334248224895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,2047,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,16383,0.016751999656359356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,16383,0.030447999636332195
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,8191,0.013717333475748697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,32767,0.03309866786003113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1,0.008703999842206636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,1,128,1,float16,fp8,32767,0.022869333624839783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,3,0.008650666723648706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,15,0.010858666151762009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,3,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1,0.010714666297038397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,15,0.008592000231146812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,7,0.00878399983048439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,7,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,31,0.008762666955590248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,63,0.00902399979531765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,127,0.008965333302815756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,31,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,63,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,127,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,255,0.011754666765530905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,511,0.012533333152532578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,511,0.011749333391586939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1023,0.013178666432698568
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,1023,0.011535999675591787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,255,0.012229333321253458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,2047,0.014271999398867289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,4095,0.01676799977819125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,2047,0.012015999605258306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,8191,0.021407999098300934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,4095,0.012367999802033106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,8191,0.013749333719412485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,16383,0.0351200004418691
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,16383,0.01692266638080279
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,32767,0.04041066765785217
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1,0.00855466661353906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,2,128,1,float16,fp8,32767,0.023376000424226124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,3,0.010725333044926325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,7,0.008816000074148178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,15,0.00878399983048439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,7,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,3,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,31,0.008810666700204214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,15,0.01062400018175443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,63,0.008879999940594038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,31,0.010858666151762009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,127,0.009183999771873156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,255,0.012234666695197424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,63,0.010832000523805618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,511,0.012576000144084295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,255,0.011706666400035223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,127,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,511,0.011813333878914515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,2047,0.014757333944241205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1023,0.013248000293970108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,1023,0.01191466674208641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,2047,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,4095,0.017290666699409485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,8191,0.014453332871198654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,16383,0.017450666675964992
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,4095,0.012618667135636011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,16383,0.03121600051720937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,8191,0.02645866572856903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,32767,0.04137066751718521
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1,0.00873066671192646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,4,128,1,float16,fp8,32767,0.024256000916163128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,3,0.01129066695769628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,15,0.008752000207702318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,3,0.008650666723648706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,7,0.011045332998037338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,31,0.008863999818762144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,7,0.008874666566650072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,15,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,63,0.00898133342464765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,31,0.011242666592200598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,127,0.009216000015536943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,127,0.011402666568756104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1023,0.013408000270525614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,511,0.012746666868527731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,63,0.011642667154471079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,255,0.012437333663304647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,255,0.01221866657336553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,511,0.01394133393963178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,2047,0.015082667271296183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,1023,0.015226667126019796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,4095,0.021770666042963665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,8191,0.026880001028378803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,4095,0.01568000018596649
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,16383,0.036373332142829895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,8191,0.018245333184798557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,2047,0.014677333335081736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,16383,0.023386667172114056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,32767,0.2007360061009725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,24,8,128,1,float16,fp8,32767,0.03417599946260452
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1,0.008725333337982496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,7,0.00871999996403853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,1,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,3,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,3,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,15,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,15,0.010672000547250112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,7,0.010682666053374609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,31,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,31,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,63,0.008997333546479544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,127,0.009109333157539368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,63,0.010890666395425797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,127,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,255,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,511,0.012762666990359625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,255,0.011616000284751257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,511,0.01163200040658315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,1023,0.011855999628702799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1023,0.013295999417702356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,2047,0.014442666123310724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,2047,0.01191466674208641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,4095,0.016938666502634685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,4095,0.01333333303531011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,8191,0.016613333175579708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,8191,0.02189333240191142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,16383,0.02254933367172877
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,16383,0.03568000098069509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,32767,0.040720000863075256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,1,128,1,float16,fp8,32767,0.03501333296298981
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1,0.012080000092585882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1,0.010757333288590113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,3,0.012128000458081564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,3,0.01081066702802976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,7,0.012223999947309494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,15,0.012229333321253458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,7,0.010751999914646149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,15,0.010794666906197866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,31,0.012213333199421564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,63,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,31,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,63,0.010842667271693548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,127,0.012608000387748083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,127,0.010981333752473196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,255,0.014773332824309668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,511,0.01524266724785169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,255,0.01163200040658315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1023,0.015669333438078564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,511,0.011722666521867117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,2047,0.01700266698996226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,1023,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,2047,0.012245333443085352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,4095,0.01979200045267741
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,4095,0.013669333110253016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,8191,0.028773332635561626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,8191,0.016864000509182613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,16383,0.034058667719364166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,16383,0.0229066660006841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,32767,0.043706665436426796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,2,128,1,float16,fp8,32767,0.035029334326585136
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1,0.0120319997270902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,3,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1,0.010960000256697336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,7,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,3,0.010970667004585266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,7,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,15,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,31,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,15,0.010960000256697336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,63,0.012426666915416718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,31,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,127,0.01258133351802826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,63,0.010954666882753372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,255,0.01492799942692121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,511,0.015301333119471868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,255,0.011893333246310553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,511,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1023,0.016048000504573185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,1023,0.012037333101034164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,2047,0.01770666614174843
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,2047,0.012597333639860153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,4095,0.024527999262015026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,4095,0.01423466702302297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,8191,0.029365333418051403
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,8191,0.01735466718673706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,16383,0.03922666609287262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,16383,0.02384000023206075
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,4,128,1,float16,fp8,32767,0.03643733263015747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1,0.012074666718641916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,7,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,3,0.011418666690587997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,3,0.012149333953857422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,32767,0.22484799226125082
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,31,0.012410666793584824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,7,0.011450666934251785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,15,0.012202666451533636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,15,0.011855999628702799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,63,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,31,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,127,0.012639999389648438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,63,0.011424000064531961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1023,0.016634666671355564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,255,0.014885333677132925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,511,0.014165333161751429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,511,0.015360000232855478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,255,0.014192000031471252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,127,0.012223999947309494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,2047,0.022218666970729828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,4095,0.027269333600997925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,1023,0.014607999473810196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,8191,0.037104000647862755
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,4095,0.018079999834299088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,2047,0.015504000087579092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,16383,0.20225600401560465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,16383,0.03329599897066752
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,8191,0.023605334262053173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,24,8,128,1,float16,fp8,32767,0.19383466243743896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,32767,0.42902398109436035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1,0.009317333499590555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,1,0.012170666207869848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,3,0.009381333366036415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,3,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,7,0.009482666850090027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,7,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,15,0.009455999980370203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,31,0.009589333087205887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,15,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,31,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,63,0.009733333562811216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,63,0.01249066616098086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,127,0.010144000252087912
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,255,0.014149333039919535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,255,0.01434133326013883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,127,0.012624000509579977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,511,0.019866666446129482
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,511,0.017130666722853977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1023,0.025018667181332905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,1023,0.02332799881696701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1,0.013242666920026144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,2047,0.034448000291983284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,2047,0.03527999917666117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,3,0.012400000045696894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,3,0.013397333522637686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,1,128,1,float16,fp8,4095,0.05937066674232483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,4095,0.19899199406305948
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,7,0.013381333400805792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,31,0.01357866699496905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,7,0.012458667159080505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,15,0.013408000270525614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,15,0.012666666259368261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,31,0.012421333541472753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,127,0.014303999642531076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,127,0.01320533330241839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,255,0.021829334398110706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,255,0.014677333335081736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,63,0.013877333452304205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,511,0.017808000246683758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,511,0.02703999976317088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,63,0.012661332885424295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1023,0.03651199986537298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,4095,0.38149865468343097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,1023,0.02407466620206833
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1,0.01452800010641416
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,4095,0.19410133361816406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1,0.017093333105246227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,3,0.01711999997496605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,2,128,1,float16,fp8,2047,0.03641066700220108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,2047,0.19917333126068115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,3,0.014560000350077948
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,7,0.01443733274936676
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,7,0.01743999992807706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,15,0.017407999684413273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,15,0.014570667097965876
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,31,0.017344000438849132
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,63,0.015125333021084467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,127,0.01882133384545644
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,127,0.015989333391189575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,255,0.018725333114465077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,255,0.023914667467276256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,31,0.014490666488806406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,63,0.017711999515692394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,511,0.02385599911212921
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,511,0.03357866654793421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,2047,0.3880853255589803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,2047,0.19671465953191122
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,1023,0.033999999364217125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1023,0.20096000035603842
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,4095,0.7583999633789062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1,0.02051199972629547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1,0.024005333582560223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,4,128,1,float16,fp8,4095,0.3741600116093953
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,3,0.024400000770886738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,7,0.02048533285657565
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,3,0.020581333587567013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,31,0.024959998826185863
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,15,0.020448000480731327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,15,0.024720000723997753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,31,0.020501332978407543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,7,0.02470933397610982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,63,0.025445332129796345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,63,0.021151999632517498
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,127,0.027109332382678986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,127,0.022181332111358643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,255,0.027514666318893433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,255,0.03708266715208689
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1023,0.38896532853444415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,511,0.202074666817983
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,1023,0.19550933440526327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,2047,0.7569173177083334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,1,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,511,0.03702933341264725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,2047,0.3760426839192708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,3,0.008623999853928884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,3,0.011045332998037338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,24,8,128,1,float16,fp8,4095,0.7318133513132731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,7,0.011007999380429586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,15,0.011055999745925268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,15,0.008757333581646284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1,0.00874133345981439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,31,0.008874666566650072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,31,0.011029332876205444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,63,0.009029333169261614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,127,0.011231999844312668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,7,0.008703999842206636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,255,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,127,0.009093333035707474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,4095,1.4945440292358398
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,255,0.011920000116030375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,511,0.012741333494583765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1023,0.013381333400805792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,511,0.011733333269755045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,1023,0.012042666474978128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,2047,0.013471999516089758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,2047,0.01482133318980535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,8191,0.026687999566396076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,8191,0.02266666789849599
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,16383,0.03169599920511246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,4095,0.017642666896184284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,16383,0.03457066665093104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,4095,0.01653333380818367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,32767,0.041696002086003624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1,0.012202666451533636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1,0.011120000233252844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,7,0.012234666695197424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,1,128,1,float16,fp8,32767,0.058464000622431435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,15,0.012122667084137598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,7,0.010933333386977514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,15,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,3,0.012133333832025528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,31,0.012282667060693106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,63,0.012448000411192576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,31,0.01109333336353302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,3,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,127,0.012624000509579977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,63,0.011061333119869232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,255,0.014869333555301031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,255,0.011973333855470022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,511,0.01534933348496755
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,511,0.012053333222866058
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1023,0.015978666643301647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,2047,0.017642666896184284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,1023,0.012144000579913458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,4095,0.02460266649723053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,4095,0.01685333376129468
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,2047,0.013616000612576803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,8191,0.029701332251230877
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,8191,0.023034666975339253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,16383,0.03946666667858759
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,16383,0.034832000732421875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1,0.012133333832025528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,3,0.012357333054145178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,2,128,1,float16,fp8,32767,0.05896000067392985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1,0.011109333485364914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,32767,0.19901333252588907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,3,0.011157333850860596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,7,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,15,0.012245333443085352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,7,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,63,0.012410666793584824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,31,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,15,0.011338666081428528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,31,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,63,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,127,0.012682666381200155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,255,0.014831999937693277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,127,0.011578666667143503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,511,0.015493333339691162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1023,0.016517333686351776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,255,0.01209066684047381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,511,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,1023,0.012645332763592402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,2047,0.022272000710169475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,2047,0.014293332894643148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,4095,0.02736533433198929
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,8191,0.03717333326737086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,4095,0.017349333812793095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,8191,0.023573334018389385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,16383,0.03587199995915095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,16383,0.2025066614151001
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,32767,0.3885440031687419
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,4,128,1,float16,fp8,32767,0.19177067279815674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1,0.013050666699806849
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,7,0.0124746672809124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1,0.012240000069141388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,3,0.013002666334311167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,3,0.012133333832025528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,15,0.012421333541472753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,7,0.013104000439246496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,31,0.01249066616098086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,63,0.012373333175977072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,15,0.012997332960367203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,31,0.013167999684810638
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,127,0.012746666868527731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,63,0.013232000172138214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,255,0.015184000134468079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,127,0.013637332866589228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,511,0.015893333901961643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,255,0.014277332772811254
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1023,0.021157334248224895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,2047,0.026144000391165417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,1023,0.015397333850463232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,4095,0.040362666050593056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,511,0.014645333091417948
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,2047,0.01820266619324684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,4095,0.023242667317390442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,8191,0.20092799266179404
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,8191,0.033610666791598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,16383,0.3872799873352051
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,16383,0.19370667139689127
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,24,8,128,1,float16,fp8,32767,0.37170668443044025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,32767,0.7546079953511556
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1,0.009722666814923286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,1,0.01321600005030632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,3,0.009872000043590864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,3,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,7,0.009930666536092758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,7,0.013093333691358566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,15,0.009946666657924652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,15,0.01310933381319046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,31,0.010106666634480158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,31,0.013242666920026144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,63,0.01022933361430963
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,63,0.013365333278973898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,127,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,127,0.013983999689420065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,255,0.015285332997639975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,255,0.020581333587567013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,511,0.02565866708755493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,511,0.018496000518401463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1023,0.03509866694609324
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,1023,0.02470933397610982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1,0.017711999515692394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,2047,0.20095467567443848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,1,128,1,float16,fp8,2047,0.037317333122094475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1,0.015178666760524115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,3,0.01781333362062772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,3,0.015173333386580149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,7,0.017946666727463405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,15,0.017994667092959087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,7,0.015040000279744467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,31,0.015290666371583939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,63,0.018496000518401463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,63,0.01584533353646596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,127,0.019578666736682255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,127,0.016602666427691776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,15,0.015279999623696009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,31,0.018277333428462345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,255,0.02460266649723053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,255,0.019226666539907455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,511,0.034485332667827606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,511,0.024613333245118458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1,0.02480533222357432
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1023,0.19646400213241577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,1023,0.0347626656293869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1,0.021125334004561108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,7,0.025557334224383037
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,3,0.025226667523384094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,3,0.021269333859284718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,2047,0.37823466459910077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,15,0.021157334248224895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,7,0.021141332884629566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,2,128,1,float16,fp8,2047,0.1991999944051107
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,31,0.025637333591779072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,31,0.021370666722456615
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,15,0.025413334369659424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,63,0.02624000112215678
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,63,0.021615999440352123
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,127,0.027818667391935985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,255,0.03875733415285746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,255,0.02804800122976303
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,511,0.03814399987459183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,511,0.20390399297078451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,127,0.02295999974012375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,2047,0.3790026505788167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,2047,0.7626720269521078
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,3,0.03940266619126002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1,0.03879466652870178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1,0.03257599969704946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,4,128,1,float16,fp8,1023,0.19889599084854126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,3,0.03274133304754893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,7,0.03973866750796636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1023,0.4288373390833537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,7,0.032858667274316154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,15,0.039877332746982574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,15,0.032773333291212715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,63,0.04093866546948751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,127,0.04365866879622141
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,127,0.03384000062942505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,63,0.032799998919169106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,31,0.04014399896065394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,255,0.20433600743611655
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,255,0.04401599864164988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1023,0.7660746574401855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,511,0.1986560026804606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,511,0.39450665314992267
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,2047,1.5068373680114746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,1023,0.39637335141499835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,2047,0.7362240155537924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,24,8,128,1,float16,fp8,31,0.03298133363326391
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,1,0.01635733370979627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,1,0.01349866638580958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,7,0.013679999858140945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,7,0.016282666474580765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,15,0.01403733342885971
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,3,0.013765333841244379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,15,0.016607999801635742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,3,0.016410666207472484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,31,0.014101333916187286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,31,0.016538667182127636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,63,0.014384000251690546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,63,0.016858667135238647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,127,0.015765332927306492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,127,0.017887999614079792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,255,0.02077866718173027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,255,0.02033599962790807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,511,0.03092266619205475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,1,128,1,float16,fp8,511,0.025786665578683216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,1,0.026122666895389557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,1,0.022367998957633972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,3,0.02658133457104365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,3,0.022687998910744984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,7,0.026874666412671406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,7,0.022517333428064983
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,15,0.026816000541051228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,15,0.022554665803909302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,31,0.02693866689999898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,31,0.022634667654832203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,63,0.02757866680622101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,63,0.02312533309062322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,127,0.029311999678611755
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,127,0.02442666639884313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,255,0.039493332306543984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,255,0.02942933390537898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,511,0.20186134179433188
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,2,128,1,float16,fp8,511,0.0391839991013209
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,1,0.034314667185147606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,1,0.04035199930270513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,3,0.041008000572522484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,3,0.03403733422358831
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,7,0.041482667128245033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,7,0.03400533397992452
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,31,0.04194133480389913
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,31,0.034261333445707955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,15,0.04124800115823746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,15,0.0340639998515447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,63,0.042490666111310325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,63,0.03429866582155228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,127,0.04498666524887085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,127,0.03524799893299738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,1,0.06515199939409892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,255,0.2085813283920288
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,1,0.05359466870625814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,255,0.04640533526738485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,511,0.40214399496714276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,4,128,1,float16,fp8,511,0.20451732476552328
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,3,0.0536106675863266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,3,0.06602666775385539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,7,0.06709333260854085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,15,0.06671999891599019
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,7,0.05378133555253347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,63,0.06897599995136261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,15,0.053871999184290566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,63,0.05436799923578898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,127,0.20882133642832437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,31,0.06752533217271169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,31,0.05381333331267039
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,127,0.056464001536369324
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,255,0.4023626645406087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,255,0.21682133277257284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,1,0.021712000171343487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,1,0.025306666890780132
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,24,8,128,1,float16,fp8,511,0.3872106472651164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,511,0.8294293085734049
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,3,0.025477332373460133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,3,0.022053333620230358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,15,0.022218666970729828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,7,0.02219199885924657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,15,0.02533866713444392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,7,0.025445332129796345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,63,0.023210667073726654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,63,0.026170666019121807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,127,0.02491733431816101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,31,0.025429333249727886
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,127,0.027317332724730175
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,31,0.02277333289384842
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,255,0.034927998979886375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,1,128,1,float16,fp8,255,0.032842665910720825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,1,0.043050666650136314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,1,0.03733866661787033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,3,0.043434664607048035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,7,0.0441599984963735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,3,0.03728000074625015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,7,0.037248000502586365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,15,0.04428266485532125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,15,0.03731200098991394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,31,0.037621334195137024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,31,0.04462933540344238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,63,0.04533333579699198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,63,0.03770666569471359
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,127,0.04780800143877665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,127,0.03865066667397817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,255,0.2111519972483317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,1,0.06880533198515575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,2,128,1,float16,fp8,255,0.05036266644795736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,1,0.0580320010582606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,3,0.06993600229422252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,3,0.05796800057093302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,7,0.07101333141326904
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,7,0.0581226646900177
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,15,0.058448001742362976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,15,0.0711946686108907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,31,0.05842133363087972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,31,0.07176533341407776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,63,0.0729973316192627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,63,0.05855466425418854
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,127,0.21875733137130737
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,127,0.06071466704209646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,1,0.11915199955304463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,255,0.4161706765492757
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,4,128,1,float16,fp8,255,0.21492266654968262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,1,0.09770666559537251
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,3,0.12129066387812297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,3,0.0974079966545105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,7,0.12333333492279053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,7,0.09762133161226909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,15,0.12328533331553142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,31,0.09840533137321472
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,63,0.21915733814239502
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,63,0.09880000352859497
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,31,0.12496532996495564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,15,0.097871998945872
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,3,0.008938666433095932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1,0.00877333308259646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,127,0.41606398423512775
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,1,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,7,0.009018666421373686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,3,0.011317333827416102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,7,0.011120000233252844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,15,0.011152000476916632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,31,0.009098666409651438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,31,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,63,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,63,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,127,0.009296000003814697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,127,0.011359999577204386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,255,0.012495999534924826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,511,0.012906666845083237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,255,0.01192533348997434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,511,0.012191999703645706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1023,0.013701333353916803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,1023,0.013781332721312841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,2047,0.016773333152135212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,2047,0.01562133307258288
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,4095,0.02252800017595291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,8191,0.027701333165168762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,4095,0.022533332308133442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,8191,0.034458667039871216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,16383,0.058778668443361916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,1,128,1,float16,fp8,32767,0.10614400108655293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,15,0.008992000172535578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,16383,0.0373333344856898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,32767,0.19967466592788696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,3,0.012437333663304647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1,0.011237333218256632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,127,0.2167946696281433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,3,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,7,0.012522666404644648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,15,0.01258133351802826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,7,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,31,0.01258133351802826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,31,0.011215999722480774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,63,0.012762666990359625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,63,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,127,0.012805332740147909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,127,0.011407999942700068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,255,0.015205333630243937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,255,0.012063999970753988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,511,0.015557333827018738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,511,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1023,0.01684800038735072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,1023,0.013797332843144735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,2047,0.022431999444961548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,4095,0.02777066578467687
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,2047,0.01692266638080279
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,4095,0.022805333137512207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,8191,0.03706666578849157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,8191,0.034901333351929985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,16383,0.19845867156982422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,16383,0.05878399809201559
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,32767,0.19283199310302734
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,32767,0.3798559904098511
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,2,128,1,float16,fp8,15,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1,0.012543999900420507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,3,0.012613333761692047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,7,0.012693333129088083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,3,0.011264000087976456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,7,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,15,0.012624000509579977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,255,0.8150666554768881
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,15,0.011391999820868174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,31,0.012719999998807907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,24,8,128,1,float16,fp8,255,0.4249173402786255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,63,0.012768000364303589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,31,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,63,0.01128000020980835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,127,0.012981332838535309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,127,0.011727999895811081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,511,0.016176000237464905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,255,0.012453333785136541
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,511,0.012602667013804117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,2047,0.026421333352724712
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,1023,0.014389333625634512
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,4095,0.040709334115187325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,2047,0.0173333336909612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,4095,0.02367466688156128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,8191,0.20086399714152017
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,8191,0.03605866680542628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,16383,0.1918933391571045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,16383,0.38867199420928955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,32767,0.7569173177083334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,3,0.012560000022252401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,4,128,1,float16,fp8,32767,0.3679893414179484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1,0.013359999905029932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1,0.012714666624863943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,3,0.013359999905029932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,7,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,7,0.013232000172138214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,15,0.01333333303531011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,15,0.012784000486135483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,31,0.012821332861979803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,31,0.013317332913478216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,63,0.012917333592971167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,63,0.013381333400805792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,127,0.013295999417702356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,255,0.01591466615597407
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,127,0.013946666071812311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,255,0.014789332946141561
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,511,0.02067199970285098
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1023,0.02587733417749405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,511,0.015381333728631338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,2047,0.03555733213822047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,1023,0.01812800019979477
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,2047,0.023525332411130268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,4095,0.03377600014209747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,4095,0.20066134134928384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,8191,0.19433067242304483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,8191,0.38606401284535724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,255,0.015402667224407196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1023,0.021530665457248688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,16383,0.7532800038655599
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,16383,0.3983413378397624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,24,8,128,1,float16,fp8,32767,0.7256800333658854
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,32767,1.4870187441507976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,1,0.03201066702604294
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,1,0.03765333443880081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,3,0.0383146678407987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,7,0.03847466657559077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,7,0.032138665517171226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,3,0.03212266663710276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,15,0.03853866706291834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,15,0.032261334359645844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,31,0.039290666580200195
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,31,0.03234133372704188
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,63,0.040175999204317726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,63,0.03249066571394602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,1,0.07404266794522603
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,127,0.04254400233427683
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,1,128,1,float16,fp8,127,0.033333333830038704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,1,0.05227200190226237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,3,0.0751039981842041
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,7,0.07625066737333934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,3,0.05243733525276184
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,7,0.05239466826121012
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,15,0.07605333129564922
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,31,0.07685333490371704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,31,0.052789335449536644
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,15,0.0526506652434667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,63,0.0786186655362447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,1,0.12659200032552084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,63,0.05333333214124044
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,2,128,1,float16,fp8,127,0.056554665168126426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,127,0.23154133558273315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,1,0.09327999750773112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,3,0.1288800040880839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,3,0.09363200267155965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,7,0.13077333569526672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,15,0.1306666632493337
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,7,0.09337600072224934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,31,0.09385066231091817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,15,0.09384000301361084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,31,0.13530133167902628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,63,0.24221867322921753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,1,0.22925867636998495
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,63,0.09520533680915833
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,127,0.44417067368825275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,1,0.1721493403116862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,4,128,1,float16,fp8,127,0.21997332572937012
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,3,0.23433599869410196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,3,0.17251733938852945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,7,0.23685866594314575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,7,0.1723946730295817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,15,0.2430079976717631
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,31,0.2754720052083333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,15,0.1725119948387146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,63,0.4451306660970052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,31,0.17409066359202066
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,63,0.22111999988555908
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,127,0.844970703125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,24,8,128,1,float16,fp8,127,0.4161653518676758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,1,0.05513600011666616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,1,0.06660800178845723
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,3,0.06764266888300578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,3,0.05517866710821787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,7,0.06846400101979573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,7,0.05533866584300995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,15,0.06855999926726024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,15,0.054976001381874084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,31,0.07025066514809926
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,31,0.055258666475613914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,63,0.09100266297658284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,1,0.1372106671333313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,1,128,1,float16,fp8,63,0.056186666091283165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,1,0.09504000345865886
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,3,0.13945600390434265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,3,0.0950879951318105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,7,0.09523199995358785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,7,0.14154133200645447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,15,0.1442080040772756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,31,0.16362667083740234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,15,0.09547199805577596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,31,0.09599467118581136
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,63,0.27984533707300824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,2,128,1,float16,fp8,63,0.09854933619499207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,1,0.24329600731531778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,3,0.24787733952204385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,1,0.180458664894104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,7,0.25562665859858197
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,7,0.18041600783665976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,3,0.18066134055455527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,15,0.27641065915425617
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,15,0.18062400817871094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,31,0.3145013252894084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,31,0.18332799275716147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,4,128,1,float16,fp8,63,0.25092800458272296
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,63,0.5357653299967448
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,1,0.4495893319447835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,1,0.3384159803390503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,3,0.46326398849487305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,3,0.33795734246571857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,7,0.5018293460210165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,7,0.33799465497334796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1,0.00898133342464765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,3,0.008976000050703684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,15,0.5181653499603271
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,1,0.011349332829316458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,3,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,7,0.00897066667675972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,7,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,15,0.3421706755956014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,15,0.011322667201360067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,31,0.5542240142822266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,31,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,63,0.009162666896979014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,127,0.009269333134094873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,15,0.009183999771873156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,127,0.011461333682139715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,31,0.00922133338948091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,255,0.012784000486135483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,255,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,511,0.013306666165590286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,31,0.3557279904683431
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,511,0.013999999811251959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,63,0.011519999553759893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1023,0.014453332871198654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,1023,0.017018667111794155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,2047,0.02046400060256322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,2047,0.02306666721900304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,4095,0.03481066723664602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,8191,0.05881066620349884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,8191,0.03559466699759165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,1,128,1,float16,fp8,16383,0.10641599694887798
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1,0.012437333663304647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,16383,0.2160373330116272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1,0.011503999431928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,3,0.012661332885424295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,3,0.011407999942700068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,7,0.011551999797423681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,7,0.012842666357755661
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,15,0.012624000509579977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,15,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,63,0.012826666235923767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,63,0.01138666644692421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,31,0.011637333780527115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,31,0.012874666601419449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,4095,0.025941332181294758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,127,0.013194666554530462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,255,0.015461333096027374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,127,0.011616000284751257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,255,0.012794667234023413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,511,0.016314666718244553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,511,0.014309333016475042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,63,0.8893600304921468
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1023,0.021685334543387096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,1023,0.017301333447297413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,4095,0.041002665956815086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,2047,0.026895999908447266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,4095,0.035386666655540466
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,2047,0.02366400013367335
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,8191,0.19706666469573975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,8191,0.05922666688760122
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,16383,0.37990931669871014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1,0.011658667276302973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1,0.012629333883523941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,7,0.012874666601419449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,3,0.011461333682139715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,3,0.012847999731699625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,7,0.011514666179815928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,15,0.011541333049535751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,15,0.012847999731699625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,31,0.012869333227475485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,31,0.011519999553759893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,63,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,63,0.011621333658695221
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,127,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,255,0.016010666886965435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,127,0.012069333344697952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,255,0.013007999708255133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,511,0.021157334248224895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,511,0.014688000082969666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,2047,0.03607466568549474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1023,0.02613866577545802
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,1023,0.017818666994571686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,2047,0.024005333582560223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,4095,0.20177066326141357
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,4095,0.036517334481080375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,8191,0.3883093198140462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,8191,0.19203199942906699
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,2,128,1,float16,fp8,16383,0.19318934281667074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,4,128,1,float16,fp8,16383,0.3823839823404948
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1,0.012858666479587555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,3,0.012960000584522883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,16383,0.7563253243764242
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1,0.01358933374285698
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,3,0.013482666263977686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,24,8,128,1,float16,fp8,63,0.4549386501312256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,7,0.012949333836634954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,15,0.013210666676362356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,7,0.013514666507641474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,15,0.0136266661187013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,31,0.013130666067202887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,31,0.013552000125249227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,63,0.013317332913478216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,63,0.013829333086808523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,127,0.013904000322024027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,255,0.02081599955757459
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,127,0.014250667144854864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,1023,0.02380799998839696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1023,0.03547733277082443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,511,0.025770666698614757
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,2047,0.1999733249346415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,2047,0.03401066611210505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,511,0.01863466699918111
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,4095,0.1945013403892517
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,4095,0.4271093209584554
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,8191,0.3737226724624634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,8191,0.7537813186645508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1,0.008922666932145754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1,0.011359999577204386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,3,0.008997333546479544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,3,0.011535999675591787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,7,0.009098666409651438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,7,0.011322667201360067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,16383,1.6718346277872722
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,15,0.009141333401203156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,255,0.016085332880417507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,15,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,24,8,128,1,float16,fp8,16383,0.7260479927062988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,31,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,31,0.009093333035707474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,63,0.009205333267649015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,63,0.011493333925803503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,127,0.009434666484594345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,127,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,255,0.013130666067202887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,511,0.013904000322024027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,255,0.013242666920026144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,511,0.016255999604860943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1023,0.01923199991385142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,1023,0.022250667214393616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,2047,0.024175999065240223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,2047,0.034058667719364166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,4095,0.03829866647720337
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,8191,0.19527467091878256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,8191,0.10578667124112447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,16383,0.3804159959157308
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,16383,0.21404266357421875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,3,0.012719999998807907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,1,128,1,float16,fp8,4095,0.058143998185793556
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,3,0.011525332927703857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1,0.011370666325092316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,7,0.012805332740147909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1,0.012709333250919977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,7,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,15,0.012906666845083237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,31,0.012874666601419449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,31,0.011445333560307821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,63,0.012975999464591345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,127,0.013338666409254074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,15,0.011482667177915573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,63,0.011541333049535751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,127,0.011936000237862269
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,255,0.0161920003592968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,255,0.013493333011865616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,511,0.016415999581416447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1023,0.026373334228992462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,511,0.021205333371957142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,1023,0.02260799954334895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,4095,0.19669866561889648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,4095,0.05821866790453593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,2047,0.034671999514102936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,2047,0.0359946663180987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,8191,0.3776373465855916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,8191,0.19150932629903158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1,0.013007999708255133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1,0.011776000261306763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,7,0.013088000317414602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,2,128,1,float16,fp8,16383,0.3702453374862671
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,15,0.013237333546082178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,7,0.011621333658695221
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,15,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,3,0.013301332791646322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,3,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,31,0.013248000293970108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,16383,0.7326773007710775
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,63,0.013482666263977686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,31,0.011749333391586939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,63,0.012005332857370377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,127,0.013962666193644205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,127,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,255,0.014069333672523499
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,255,0.02089600016673406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,511,0.025983999172846477
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,2047,0.20130133628845215
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,511,0.01709866647919019
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,2047,0.035829332967599235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,1023,0.023376000424226124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,4095,0.38700799147288006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,4095,0.19222400585810342
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1023,0.035877334574858345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,8191,0.36823999881744385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,8191,0.7540480295817057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1,0.01379199946920077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,3,0.016751999656359356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1,0.016458666572968166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,4,128,1,float16,fp8,16383,0.7946133613586426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,7,0.016842667013406754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,15,0.016821333517630894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,7,0.013701333353916803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,3,0.013786666095256805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,16383,1.4886825879414876
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,31,0.01695466662446658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,15,0.013653332988421122
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,63,0.017317333569129307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,31,0.013850666582584381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,63,0.014282666146755219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,127,0.018581333259741466
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,255,0.023168000082174938
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,127,0.015423999478419622
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,255,0.017946666727463405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,511,0.03287466615438461
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1023,0.21811733643213907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,2047,0.3845813274383545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,2047,0.1941279967625936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,511,0.02332799881696701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,1023,0.033733333150545754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,4095,0.7509333292643229
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,4095,0.37170132001241046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1,0.008559999987483025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1,0.010543999572594961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,3,0.008639999975760778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,8191,0.7254559993743896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,7,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,3,0.010591999938090643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,7,0.010645333677530289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,15,0.008645333349704742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,8191,1.6547306378682454
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,15,0.010735999792814255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,31,0.008645333349704742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,31,0.0106133334338665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,63,0.008650666723648706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,127,0.00898133342464765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,63,0.010527999450763067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,127,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,255,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,511,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,255,0.011402666568756104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1023,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,1023,0.011600000162919363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,511,0.011391999820868174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,2047,0.014175999909639359
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,2047,0.01184533288081487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,4095,0.016437333077192307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,16383,3.328282674153646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,4095,0.01210133358836174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,8191,0.013466666142145792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,8191,0.020810666183630627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,16383,0.030095999439557392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,16383,0.01657066618402799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,32767,0.02269333352645238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,65535,0.0379573330283165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,16,8,128,1,float16,fp8,16383,1.4334826469421387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1,0.008394666636983553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,1,128,1,float16,fp8,65535,0.03313066562016805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,3,0.008576000109314919
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,32767,0.03310399999221166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,3,0.010538666198650995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,7,0.010538666198650995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,15,0.008527999743819237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,15,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,31,0.008693333094318708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,63,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,31,0.010677333921194077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,127,0.008874666566650072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,63,0.010629333555698395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,127,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,255,0.012240000069141388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,255,0.011551999797423681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1023,0.013034666577974955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,7,0.00877333308259646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,511,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,511,0.012565333396196365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,1023,0.011658667276302973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,2047,0.014250667144854864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,2047,0.011855999628702799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,4095,0.01231466606259346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,8191,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,4095,0.01676799977819125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,16383,0.016864000509182613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,32767,0.040037333965301514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,16383,0.035018667578697205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,8191,0.021514666577180225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,32767,0.022976001103719074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,65535,0.05006400247414907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,2,128,1,float16,fp8,65535,0.03348266581694285
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,3,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,15,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,7,0.008949333180983862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,7,0.010634666929642359
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,3,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1,0.008629333227872849
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,31,0.008863999818762144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,15,0.010687999427318573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,31,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,63,0.00895999992887179
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,63,0.010608000059922537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,255,0.012245333443085352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,127,0.008992000172535578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,511,0.012469333906968435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,255,0.01157333329319954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,127,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1023,0.01313599944114685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,511,0.011701333026091257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,4095,0.0173333336909612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,1023,0.011850666254758835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,8191,0.02624000112215678
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,2047,0.012015999605258306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,2047,0.014688000082969666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,8191,0.014384000251690546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,4095,0.012949333836634954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,16383,0.031141333281993866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,32767,0.040965333580970764
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,16383,0.01764800027012825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,32767,0.023823998868465424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,65535,0.20462934176127115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,4,128,1,float16,fp8,65535,0.0347680002450943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,3,0.00867733359336853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1,0.01118933285276095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1,0.008682666967312494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,3,0.011301333705584208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,7,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,7,0.008736000085870424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,15,0.012159999459981918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,15,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,31,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,31,0.011109333485364914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,63,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,127,0.008965333302815756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,63,0.00890666681031386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,127,0.011690666278203329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,511,0.012719999998807907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,255,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,255,0.012373333175977072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,511,0.013951999445756277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,2047,0.014959999670584997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1023,0.013434667140245438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,4095,0.02182399978240331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,2047,0.014602666099866232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,1023,0.014074667046467463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,16383,0.03633599976698557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,8191,0.018170667191346485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,4095,0.015647999942302704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,8191,0.026767998933792114
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,32767,0.19993066787719727
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,16383,0.02350933353106181
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,32767,0.03408000121514002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,65535,0.3892693519592285
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1,0.008639999975760778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1,0.010847999403874079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,16,8,128,1,float16,fp8,65535,0.19273600975672403
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,3,0.008752000207702318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,3,0.010677333921194077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,7,0.00877333308259646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,15,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,7,0.010645333677530289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,31,0.009375999992092451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,15,0.01062400018175443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,31,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,63,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,63,0.010677333921194077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,127,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,127,0.010960000256697336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,255,0.011658667276302973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,511,0.012645332763592402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,255,0.01231466606259346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1023,0.013269333789745966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,511,0.01181866725285848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,1023,0.01173866664369901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,2047,0.014266667266686758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,2047,0.011871999750534693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,4095,0.016901332885026932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,4095,0.013594667116800943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,8191,0.021738665799299877
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,16383,0.03518400092919668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,8191,0.016528000434239704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,16383,0.022442666192849476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,32767,0.04014399896065394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,32767,0.03474666674931844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,1,128,1,float16,fp8,65535,0.057002668579419456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1,0.010698666175206503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1,0.011887999872366587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,65535,0.05026133358478546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,3,0.01201066623131434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,7,0.012159999459981918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,3,0.010586666564146677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,15,0.012069333344697952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,7,0.010757333288590113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,31,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,15,0.010709332923094431
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,31,0.010677333921194077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,63,0.01231466606259346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,63,0.010687999427318573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,127,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,255,0.014869333555301031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,255,0.011685332904259363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,511,0.01498666654030482
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1023,0.015909332782030106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,4095,0.019786667078733444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,2047,0.017045332739750545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,1023,0.01157333329319954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,2047,0.01239466667175293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,4095,0.013530666629473368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,8191,0.028864001234372456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,511,0.011834666132926941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,16383,0.03386666625738144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,8191,0.01661866654952367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,16383,0.022672000030676525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,32767,0.043882668018341064
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,32767,0.034815999368826546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1,0.012149333953857422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,65535,0.20228266716003418
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,3,0.012159999459981918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,3,0.010890666395425797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,2,128,1,float16,fp8,65535,0.057317331433296204
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,7,0.012128000458081564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,15,0.012245333443085352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,31,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,7,0.0107893335322539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,15,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,63,0.012426666915416718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,63,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,127,0.012655999511480331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,255,0.014783999572197596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,127,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,31,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,255,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,511,0.015317333241303762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,511,0.01181866725285848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1023,0.01602666700879733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,2047,0.01756799966096878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,4095,0.024304000039895374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,1023,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,8191,0.029365333418051403
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,4095,0.01443733274936676
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,2047,0.012560000022252401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,8191,0.01740266631046931
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,16383,0.03899733225504557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,16383,0.02347733328739802
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,32767,0.20400534073511759
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,32767,0.03621866554021835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,3,0.012181332955757776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,65535,0.3942026694615682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1,0.012170666207869848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,4,128,1,float16,fp8,65535,0.19164266188939413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,7,0.012138667205969492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,3,0.011733333269755045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,63,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,31,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,7,0.011653333902359009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,15,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,15,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,63,0.01157333329319954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,31,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,127,0.012469333906968435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,255,0.014842666685581207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,255,0.013989333063364029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,127,0.011802667131026586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,511,0.01422400027513504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,2047,0.0220320001244545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,511,0.015482666591803232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,1023,0.014309333016475042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,4095,0.027263998985290527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1023,0.016501333564519882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,8191,0.036890665690104164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,2047,0.015493333339691162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,4095,0.018101333330074947
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,16383,0.20130133628845215
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,8191,0.02334933231274287
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,16383,0.033589333295822144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,32767,0.19340799252192178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,32767,0.42928000291188556
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,16,8,128,1,float16,fp8,65535,0.3707679907480876
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1,0.00902399979531765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,65535,0.8442826271057129
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1,0.012042666474978128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,3,0.00926399976015091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,3,0.01210133358836174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,7,0.009152000149091085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,7,0.012085333466529846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,15,0.009269333134094873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,15,0.011893333246310553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,31,0.009573333586255709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,31,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,63,0.009408000235756239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,127,0.010079999764760336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,63,0.011978667229413986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,255,0.013834666460752487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,127,0.012410666793584824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,255,0.014032000054915747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,511,0.019199999670187633
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,511,0.017008000363906223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,1023,0.02309333284695943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1023,0.023925334215164185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,8191,0.19477333625157675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,8191,0.38097600142161053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,2047,0.034976000587145485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,2047,0.033759998778502144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1,0.012069333344697952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,3,0.012949333836634954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,1,128,1,float16,fp8,4095,0.05888533095518748
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,4095,0.1967573364575704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,3,0.012047999848922094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,7,0.012170666207869848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,7,0.012960000584522883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,15,0.013237333546082178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,15,0.012085333466529846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,63,0.013397333522637686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,63,0.012346666306257248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,127,0.01394133393963178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,127,0.012800000607967377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,31,0.013242666920026144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,255,0.021295999487241108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,31,0.011973333855470022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,255,0.01431999976436297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,511,0.026250667870044708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,511,0.01735466718673706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1023,0.03611200054486593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,1023,0.023786666492621105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,4095,0.3790666659673055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,4095,0.19356799125671387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,2047,0.1974666714668274
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,2047,0.03618666778008143
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1,0.014021333307027817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,8191,0.7338399887084961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,3,0.01681600014368693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1,0.01659199967980385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,2,128,1,float16,fp8,8191,0.36899733543395996
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,7,0.014106666048367819
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,3,0.014309333016475042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,31,0.01706133286158244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,15,0.014085333794355392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,31,0.014293332894643148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,15,0.01676799977819125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,63,0.017237332959969837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,7,0.01687466725707054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,127,0.01851733277241389
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,127,0.015610666324694952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,63,0.014805333067973455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,255,0.023525332411130268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,255,0.018351999421914417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,511,0.032858667274316154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,511,0.023333333432674408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,2047,0.195360004901886
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,2047,0.3869280020395915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,4095,0.37380798657735187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,4095,0.7575360139211019
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,1023,0.03364266703526179
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1023,0.1998293399810791
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1,0.020202666521072388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1,0.023792001108328503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,3,0.02421333392461141
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,3,0.020309332758188248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,15,0.024480000138282776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,4,128,1,float16,fp8,8191,0.7275306383768717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,15,0.020213333268960316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,31,0.024773334463437397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,8191,1.494991938273112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,31,0.020303999384244282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,7,0.020224000016848247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,7,0.024400000770886738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,63,0.025040000677108765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,127,0.026719999810059864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,63,0.0206986665725708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,127,0.022090665996074677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,255,0.02714666724205017
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1023,0.3877546787261963
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,255,0.0364479993780454
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,1023,0.19489065806070963
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,2047,0.3752266565958659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,2047,0.8403306802113851
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,511,0.03697066754102707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,511,0.2197279930114746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1,0.008639999975760778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1,0.010847999403874079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,7,0.008842666943868002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,4095,0.7305760383605957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,7,0.010938666760921478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,15,0.00873066671192646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,3,0.008645333349704742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,15,0.01102399950226148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,31,0.008885333314538002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,31,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,63,0.00891733355820179
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,16,8,128,1,float16,fp8,8191,1.4385652542114258
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,63,0.01099733387430509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,127,0.008901333436369896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,4095,1.4939786593119304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,255,0.01231466606259346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,127,0.011125333607196808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,3,0.010901333143313726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,255,0.011877333124478659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,2047,0.014650666465361914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,1023,0.01209066684047381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,511,0.011749333391586939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,511,0.01259200026591619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1023,0.013280000537633896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,2047,0.013408000270525614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,4095,0.017375999440749485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,4095,0.01669866715868314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,16383,0.034448000291983284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,8191,0.022389332453409832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,8191,0.02638400097688039
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,16383,0.0315786674618721
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,32767,0.041365332901477814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,32767,0.05841066439946493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,1,128,1,float16,fp8,65535,0.10594666997591655
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,65535,0.22052266200383505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,3,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,3,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1,0.01102399950226148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,7,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,15,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,15,0.010933333386977514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,31,0.012362666428089142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,7,0.01089599976936976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,63,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,31,0.01102399950226148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,8191,2.9639199574788413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,63,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,127,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,255,0.01488000030318896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,511,0.015301333119471868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,127,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,255,0.01184533288081487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1023,0.016095999628305435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,511,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,1023,0.01246400053302447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,2047,0.017551999539136887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,4095,0.024314666787783306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,2047,0.013765333841244379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,4095,0.016682667036851246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,8191,0.029626667499542236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,16383,0.03952533255020777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,8191,0.022645334402720135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,16383,0.03465066601832708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,32767,0.19876799980799356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,32767,0.058874666690826416
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,2,128,1,float16,fp8,65535,0.1920213301976522
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1,0.01221866657336553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,65535,0.38202667236328125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,3,0.012149333953857422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,3,0.011109333485364914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,15,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,7,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,7,0.011173332730929056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,31,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,15,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,31,0.011120000233252844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,63,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,63,0.012234666695197424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,127,0.012746666868527731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,255,0.015077333897352219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,127,0.01145600030819575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,255,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,511,0.01543466622630755
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,2047,0.022069332500298817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,511,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,4095,0.027232001225153606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,2047,0.014208000153303146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,1023,0.012655999511480331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1023,0.016506666938463848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,4095,0.01724799970785777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,8191,0.03700799991687139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,8191,0.023765332996845245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,16383,0.2020960052808126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,16383,0.0358240008354187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,32767,0.386954665184021
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,32767,0.1906986633936564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,3,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1,0.012965332716703415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,4,128,1,float16,fp8,65535,0.36856532096862793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,65535,0.757525364557902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,15,0.012378666549921036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,3,0.013125333935022354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,7,0.013007999708255133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,7,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,31,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,15,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,63,0.01240533341964086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,31,0.013013333082199097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,127,0.012730666746695837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,63,0.012970666090647379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,255,0.015173333386580149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,511,0.016106666376193363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,255,0.01440000037352244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,127,0.013728000223636627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1023,0.020960000654061634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,511,0.014554666976133982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,4095,0.040261333187421165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,1023,0.015482666591803232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,2047,0.018085333208243053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,2047,0.026170666019121807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,8191,0.2002133329709371
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,4095,0.023306667804718018
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,8191,0.033728001018365227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,16383,0.38651732603708905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,16383,0.20811732610066733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,32767,0.3707520167032878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,32767,0.7540266513824463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,16,8,128,1,float16,fp8,65535,0.7807253201802572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,65535,1.4920053482055664
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1,0.009589333087205887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1,0.012549333274364471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,3,0.012789333860079447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,3,0.009775999933481216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,7,0.009765333185593287
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,7,0.01258133351802826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,15,0.009759999811649323
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,15,0.012794667234023413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,31,0.009765333185593287
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,31,0.012853333105643591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,63,0.012837332983811697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,63,0.010170666500926018
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,127,0.013280000537633896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,127,0.010687999427318573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,255,0.01966399947802226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,255,0.014906667172908783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,511,0.01809599995613098
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,511,0.02470933397610982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1023,0.03431999931732813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,1023,0.024330665667851765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1,0.017301333447297413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1,0.014682666709025701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,2047,0.03692800054947535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,3,0.01738133281469345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,2047,0.19840532541275024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,3,0.01469333345691363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,7,0.017514667163292568
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,7,0.014741333822409311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,4095,0.3818560043970744
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,1,128,1,float16,fp8,4095,0.195850670337677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,15,0.014794666320085526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,15,0.01754133279124896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,31,0.014848000059525171
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,31,0.01758933315674464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,63,0.018138666947682697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,63,0.015311999867359797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,255,0.024154665569464367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,255,0.018895999838908512
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,511,0.033786666889985405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,511,0.02404800057411194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,127,0.018901333212852478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1023,0.1946986714998881
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,1023,0.0341386670867602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,4095,0.7321279843648275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,127,0.016042667130629223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,4095,0.3742719888687134
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1,0.024282666544119518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1,0.020746666938066483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,3,0.02062400057911873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,7,0.02476799984773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,2,128,1,float16,fp8,2047,0.1971679925918579
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,2047,0.3758399883906047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,7,0.020645332833131153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,15,0.02474133421977361
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,3,0.024613333245118458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,15,0.020794666061798733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,31,0.025093334416548412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,31,0.020848001043001812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,127,0.022458667556444805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,255,0.03687999894221624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,127,0.027263998985290527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,255,0.02759466568628947
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,511,0.20241600275039673
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,2047,0.7590826352437338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,63,0.025642665723959606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,63,0.021221332252025604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,511,0.03716266651948293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,2047,0.377674659093221
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1,0.03809600075085958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1,0.03233066697915395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,4095,0.7887093226114908
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,4,128,1,float16,fp8,1023,0.19701866308848062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1023,0.3905920187632243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,3,0.03236266722281774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,7,0.03920000046491623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,4095,1.4996639887491863
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,7,0.0323840007185936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,3,0.0386559988061587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,15,0.03909866760174433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,31,0.032485333581765495
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,63,0.04035199930270513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,63,0.032618666688601174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,127,0.043050666650136314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,127,0.03323733309904734
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,15,0.03241066634654999
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,31,0.03962666789690653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,255,0.20246400435765585
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,255,0.043791999419530235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,1023,0.37719468275705975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1023,0.7628373305002848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,2047,0.7339680194854736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,2047,1.6720213890075684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,511,0.19659733772277832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,511,0.39211734135945636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,16,8,128,1,float16,fp8,4095,1.443845272064209
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1,0.013503999759753546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1,0.015471999843915304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,3,0.013514666507641474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,4095,2.983791987101237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,3,0.01568000018596649
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,15,0.013786666095256805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,7,0.01565333331624667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,15,0.015520000209410986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,7,0.013663999736309052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,31,0.013893333574136099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,31,0.015664000064134598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,63,0.014202666779359182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,63,0.0161920003592968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,127,0.01699200024207433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,255,0.02021866664290428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,127,0.015306666493415833
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,255,0.019738666713237762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,511,0.030213333666324615
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,511,0.02473066747188568
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1,0.025205334027608235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,3,0.02569599946339925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1023,0.19420266151428223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1,0.02164799968401591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,1,128,1,float16,fp8,1023,0.03526933242877325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,3,0.02161066730817159
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,7,0.025994665920734406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,15,0.025813333690166473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,15,0.02163733293612798
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,31,0.021717332303524017
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,7,0.021503999829292297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,63,0.022154666483402252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,31,0.02629866699377696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,63,0.026901334524154663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,127,0.02850666642189026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,127,0.023247999449570973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,255,0.038480001191298165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,255,0.02867199977238973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,511,0.03809600075085958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1,0.03941866755485535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,511,0.1979573369026184
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1,0.03314133236805598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1023,0.3808586597442627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,2,128,1,float16,fp8,1023,0.20220800240834555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,3,0.040021332601706185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,3,0.033344000577926636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,7,0.03342399994532267
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,7,0.040474665661652885
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,15,0.04060266663630804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,15,0.03315199911594391
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,63,0.04180799921353658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,63,0.03357866654793421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,127,0.044250667095184326
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,127,0.03451200077931086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,31,0.041077333192030586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,31,0.03331733246644338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,255,0.20546666781107584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,255,0.044495999813079834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1,0.06413866579532623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1,0.0528053343296051
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,511,0.20015466213226318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,511,0.3970346848169963
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,3,0.0655680000782013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,7,0.06633600095907848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,3,0.05258133510748545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,7,0.05282666782538096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,4,128,1,float16,fp8,1023,0.38262399037679035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1023,0.777733325958252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,31,0.06730133295059204
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,31,0.05293866495291392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,15,0.06612800061702728
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,15,0.05268799761931101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,63,0.0682826687892278
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,63,0.05356266597906748
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,1023,0.7407360076904297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1023,1.5054240226745605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,255,0.39738667011260986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,127,0.05568000177542368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,255,0.20097066958745322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,127,0.20612800121307373
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,1,0.02123733361562093
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,1,0.023391999304294586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,16,8,128,1,float16,fp8,511,0.383407990137736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,511,0.7737493515014648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,3,0.021333334346612293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,7,0.023503998915354412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,3,0.023728000621000927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,7,0.021594665944576263
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,15,0.02179199953873952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,15,0.023381332556406658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,63,0.022442666192849476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,63,0.02418133368094762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,127,0.024570666253566742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,127,0.025568000972270966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,31,0.023647998770078022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,31,0.021930667261282604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,255,0.03054400036732356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,255,0.034261333445707955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,511,0.19905600945154825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,1,0.04147200038035711
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,1,128,1,float16,fp8,511,0.040607998768488564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,1,0.03509333233038584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,3,0.04177066683769226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,3,0.03499733408292135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,7,0.042725334564844765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,15,0.042538667718569435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,7,0.0351946676770846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,31,0.04298666616280874
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,31,0.03518400092919668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,15,0.03513066718975703
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,63,0.04374399781227112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,63,0.035717333356539406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,127,0.04595733185609182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,127,0.03655466685692469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,255,0.20458134015401205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,1,0.06714666883150737
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,1,0.05628266433874766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,511,0.393338680267334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,3,0.0680320014556249
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,511,0.2097866733868917
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,3,0.05624533196290334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,7,0.06941866874694824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,7,0.05613866448402405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,15,0.06925333539644878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,31,0.07008000214894612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,2,128,1,float16,fp8,255,0.04809066653251648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,63,0.07189866900444031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,31,0.056330665946006775
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,15,0.05602133274078369
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,63,0.05681066711743673
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,127,0.21335466702779135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,127,0.05913599828879038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,1,0.11746666828791301
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,255,0.20841066042582193
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,1,0.09540266791979472
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,255,0.4078559875488281
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,3,0.11981333295504253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,511,0.8104213078816732
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,4,128,1,float16,fp8,511,0.39217066764831543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,7,0.12120532989501953
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,3,0.09573333462079366
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,15,0.12126933534940083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,31,0.12325333555539449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,15,0.09573866923650105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,7,0.0956213374932607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,63,0.09670933087666829
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,63,0.21329599618911743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,31,0.09634666641553243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,127,0.2113493283589681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,127,0.446394681930542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1,0.009925333162148794
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,7,0.008912000184257826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,7,0.011114666859308878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,15,0.011168000598748526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,15,0.008869333192706108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,31,0.008986666798591614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,31,0.011231999844312668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,63,0.009098666409651438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,63,0.011215999722480774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,127,0.009253333633144697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,127,0.01146666705608368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,3,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,255,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,255,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,511,0.012154666086037954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,3,0.008976000050703684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1023,0.01350933313369751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,511,0.012714666624863943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,1023,0.013807999591032663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,2047,0.016565332810084026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,2047,0.015184000134468079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,4095,0.022709332406520844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,4095,0.022474666436513264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,8191,0.027290667096773785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,255,0.7828160127003988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,8191,0.03461866577466329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,16383,0.03712533414363861
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,32767,0.10611200332641602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,32767,0.19758933782577515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,16383,0.05874133110046387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1,0.012037333101034164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,1,128,1,float16,fp8,65535,0.2143519918123881
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1,0.011114666859308878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,3,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,3,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,7,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,7,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,15,0.012560000022252401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,15,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,255,0.3939359982808431
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,31,0.012432000289360682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,31,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,63,0.012522666404644648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,63,0.011152000476916632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,127,0.011418666690587997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,127,0.012725333372751871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,255,0.012144000579913458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,511,0.015370666980743408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,511,0.012367999802033106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1023,0.016522667060295742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,1023,0.013770667215188345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,2047,0.022389332453409832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,4095,0.02743999908367793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,2047,0.01676799977819125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,8191,0.03728000074625015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,4095,0.022991999983787537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,8191,0.03489600121974945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,16383,0.05885866781075796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,16383,0.19690666596094766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,32767,0.37789865334828693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,32767,0.19297067324320474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,65535,0.7358933289845785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1,0.012191999703645706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,2,128,1,float16,fp8,65535,0.37089598178863525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,3,0.012421333541472753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,7,0.012346666306257248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,3,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,7,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,15,0.012346666306257248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,65535,0.3833119869232178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,15,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,31,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,255,0.015013333410024643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,63,0.012538666526476542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,31,0.011349332829316458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,63,0.011322667201360067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,127,0.012800000607967377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,127,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,255,0.01516266663869222
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,255,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,511,0.016010666886965435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,511,0.012752000242471695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,2047,0.02605333427588145
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1023,0.021114667256673176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,2047,0.017360000560681026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,4095,0.04016000032424927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,1023,0.014416000495354334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,4095,0.023754666248957317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,8191,0.20121065775553384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,8191,0.036133334040641785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,16383,0.38737066586812335
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,16383,0.19180800517400107
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,32767,0.3673119942347209
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1,0.012362666428089142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1,0.013295999417702356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,511,1.6908373832702637
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,32767,0.7546239693959554
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,3,0.013189333180586496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,3,0.012400000045696894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,4,128,1,float16,fp8,65535,0.7934026718139648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,16,8,128,1,float16,fp8,511,0.7530773480733236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,7,0.012448000411192576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,65535,1.4880906740824382
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,15,0.013376000026861826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,7,0.013397333522637686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,31,0.01246400053302447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,63,0.012543999900420507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,127,0.012970666090647379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,31,0.013125333935022354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,127,0.01392000044385592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,255,0.01488000030318896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,255,0.015728000551462173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1023,0.025594666600227356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,511,0.020506666352351505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,1023,0.018298666924238205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,511,0.015429332852363586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,2047,0.035274667044480644
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,4095,0.22007467349370322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,8191,0.38470399379730225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,2047,0.023589332898457844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,4095,0.03341866781314214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,8191,0.19378133614857992
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,16383,0.7536053657531738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,16383,0.37067198753356934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,32767,0.7248480319976807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,32767,1.6717653274536133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,65535,1.545610745747884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,65535,3.3305333455403647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,16,8,128,1,float16,fp8,63,0.013455999394257864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,15,0.012608000387748083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,1,0.036501333117485046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,1,0.03124266614516576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,3,0.037050666908423104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,3,0.03120533376932144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,7,0.0374293327331543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,7,0.03125333289305369
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,15,0.037274666130542755
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,15,0.03145600110292435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,31,0.03817066550254822
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,31,0.031290667752424874
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,63,0.03889599939187368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,63,0.03150933235883713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,127,0.04073066761096319
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,127,0.03271999955177307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,255,0.2084266742070516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,1,128,1,float16,fp8,255,0.043824002146720886
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,1,0.07082133491834004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,1,0.05124799907207489
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,3,0.07209066549936931
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,3,0.05119466781616211
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,7,0.07291199763615926
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,7,0.051407997806866966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,15,0.07282133400440216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,15,0.05162666738033295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,31,0.07354666789372762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,31,0.05172266562779745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,63,0.07564266522725423
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,63,0.05201066533724467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,127,0.21875200668970743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,127,0.05527466535568237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,1,0.12363200386365254
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,255,0.40959465503692627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,2,128,1,float16,fp8,255,0.20710400740305582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,1,0.09225599964459737
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,3,0.12584533294041952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,3,0.09217600027720134
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,7,0.12706133723258972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,7,0.09236799677213033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,15,0.12705600261688232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,31,0.0929813285668691
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,15,0.09291199843088786
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,31,0.12997333208719888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,63,0.22708799441655478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,63,0.09366933504740398
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,127,0.42769066492716473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,127,0.21344532569249472
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,1,0.2258560061454773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,1,0.17162134250005087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,255,0.8068426450093588
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,4,128,1,float16,fp8,255,0.4243946472803752
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,3,0.23072532812754312
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,3,0.17114132642745972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,7,0.2334666649500529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,15,0.23435733715693155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,15,0.17212265729904175
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,7,0.17153600851694742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,63,0.23170665899912515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,63,0.4282453457514445
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,31,0.2680266698201497
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,31,0.17265599966049194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,127,0.8078560034434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,127,0.405786673227946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,255,1.7172106107076008
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,16,8,128,1,float16,fp8,255,0.7688906987508138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,1,0.06403733293215434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,1,0.053642665346463524
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,3,0.06543466448783875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,3,0.053413331508636475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,7,0.06613866488138835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,7,0.05357866485913595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,31,0.06704533100128174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,63,0.06905599931875865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,15,0.053530668218930565
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,31,0.0540533314148585
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,63,0.05454400181770325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,15,0.06689066688219707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,127,0.22699199120203653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,1,128,1,float16,fp8,127,0.05807999769846598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,1,0.12987732887268066
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,1,0.09316800038019817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,3,0.1318773329257965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,7,0.0936906635761261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,3,0.09346666932106018
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,7,0.1341973344484965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,15,0.13467199603716531
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,31,0.14204800128936768
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,15,0.09366933504740398
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,31,0.093941330909729
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,63,0.249616007010142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,63,0.09519466757774353
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,2,128,1,float16,fp8,127,0.22725866238276163
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,127,0.44418664773305255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,1,0.23647467295328775
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,1,0.17779733737309775
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,3,0.24057066440582275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,3,0.17824532588322958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,7,0.24429333209991455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,7,0.17865065733591715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,15,0.256111999352773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,15,0.1790613333384196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,31,0.29024000962575275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,63,0.46644266446431476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,63,0.2518986662228902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,31,0.17946134010950723
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,1,0.4414986769358317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,127,0.8487466971079508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,4,128,1,float16,fp8,127,0.43160001436869305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,1,0.33667198816935223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,3,0.44997334480285645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,3,0.33634666601816815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,7,0.47417600949605304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,7,0.33619733651479083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,15,0.4988853136698405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,31,0.5284479856491089
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,31,0.35450132687886554
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,15,0.33859201272328693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,3,0.008912000184257826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,3,0.011157333850860596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,7,0.008965333302815756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1,0.011359999577204386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,7,0.011087999989589056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1,0.008965333302815756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,31,0.00892800030608972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,15,0.008890666688481966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,15,0.011285333583752314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,31,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,63,0.009119999905427298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,127,0.011317333827416102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,63,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,127,0.009093333035707474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,255,0.01258133351802826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,255,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,511,0.013679999858140945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,511,0.012965332716703415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1023,0.014256000518798828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,1023,0.016837333639462788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,2047,0.0225600004196167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,2047,0.020303999384244282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,4095,0.025125332176685333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,4095,0.0347680002450943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,8191,0.05871466795603434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,8191,0.03482666611671448
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,16383,0.19522666931152344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,16383,0.10636799534161885
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,1,128,1,float16,fp8,32767,0.21407999595006308
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,32767,0.3794613281885783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1,0.012458667159080505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,3,0.011349332829316458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,7,0.01250133290886879
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,7,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,3,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,63,0.8537066777547201
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,15,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,31,0.012714666624863943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,31,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,15,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,63,0.012725333372751871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,63,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,127,0.011391999820868174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,255,0.015381333728631338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,255,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,511,0.013781332721312841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,511,0.016149333367745083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1023,0.0215786670645078
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,1023,0.01682666689157486
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,4095,0.04083733260631561
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,2047,0.023034666975339253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,2047,0.026629333694775898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,4095,0.03487999985615412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,8191,0.19688000281651816
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,8191,0.05913066864013672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,16383,0.19244800011316934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,16383,0.37805334726969403
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1,0.012586666891972223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,2,128,1,float16,fp8,32767,0.3703146775563558
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1,0.01128000020980835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,32767,0.7744853496551514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,3,0.012576000144084295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,7,0.011413333316644033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,15,0.012693333129088083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,3,0.01139733319481214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,7,0.01267733300725619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,31,0.012784000486135483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,15,0.011402666568756104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,63,0.012885333349307379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,31,0.011482667177915573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,127,0.013162666310866674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,63,0.01137599969903628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,127,0.013359999905029932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,127,0.011690666278203329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,255,0.012896000097195307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,511,0.02089066555102666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,255,0.015989333391189575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,511,0.014511999984582266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1023,0.02589866767326991
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,1023,0.017664000391960144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,2047,0.03557866563399633
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,2047,0.02388266722361247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,4095,0.200981338818868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,4095,0.03590933233499527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,8191,0.19182399908701578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,8191,0.38655467828114826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,16383,0.36799466609954834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,16383,0.8349226315816244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1,0.012805332740147909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1,0.013450667262077332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,4,128,1,float16,fp8,32767,0.7216746807098389
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,3,0.012906666845083237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,32767,1.6686773300170898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,3,0.013359999905029932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,7,0.013647999614477158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,7,0.012890666723251343
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,15,0.013002666334311167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,31,0.012991999586423239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,15,0.013264000415802002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,63,0.013434667140245438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,31,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,63,0.013765333841244379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,127,0.01370666672786077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,127,0.014261333892742792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,63,0.4347039858500163
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,255,0.020794666061798733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,255,0.015781333049138386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1023,0.035402665535608925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,511,0.018640000373125076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,1023,0.0236160010099411
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,511,0.025631998976071674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,2047,0.03376533339420954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,4095,0.19394133488337198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,4095,0.3850613435109456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,8191,0.7535786628723145
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,8191,0.37249600887298584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,16383,0.7252533435821533
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,2047,0.20066134134928384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,16383,1.670032024383545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1,0.008912000184257826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,3,0.008992000172535578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,16,8,128,1,float16,fp8,127,0.8684373696645101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,16,8,128,1,float16,fp8,32767,1.5445866584777832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,3,0.011391999820868174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,7,0.009045333291093508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,7,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,15,0.011338666081428528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,15,0.009002666920423508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,32767,3.325237274169922
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,31,0.009109333157539368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,63,0.00921066664159298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,63,0.011343999455372492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,127,0.009349333122372627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,255,0.013082666943470636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,127,0.011802667131026586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,255,0.012901333471139273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,511,0.013701333353916803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,511,0.016149333367745083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,1023,0.022128000855445862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1023,0.01894933357834816
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,2047,0.024138666689395905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,2047,0.033941333492596946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,4095,0.03788266579310099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,4095,0.058101331194241844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,8191,0.19404266277949014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,8191,0.1055466632048289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1,0.011509332805871964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,16383,0.2134666641553243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,3,0.011445333560307821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,16383,0.3784000078837077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1,0.012485332787036896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,3,0.012655999511480331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,7,0.012800000607967377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,7,0.011317333827416102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,15,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,15,0.012805332740147909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,31,0.012682666381200155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,63,0.013013333082199097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,31,0.011488000551859537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,63,0.011519999553759893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,127,0.01340266689658165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,127,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,255,0.015957333147525787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,255,0.013370666652917862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,127,1.7345813115437825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,511,0.02107733239730199
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,511,0.016362667083740234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1023,0.026208000878492992
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,1023,0.0225600004196167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,4095,0.05827199916044871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,2047,0.035946667194366455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,2047,0.03456533451875051
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,8191,0.1918720006942749
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,2,128,1,float16,fp8,16383,0.37010133266448975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,1,128,1,float16,fp8,31,0.011365332951148352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,3,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,3,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,7,0.013082666943470636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1,0.012762666990359625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,16383,0.7312959829966227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,7,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,15,0.012997332960367203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,15,0.011653333902359009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,31,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,31,0.01157333329319954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,63,0.013274667163689932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,63,0.011834666132926941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,127,0.013888000200192133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,127,0.012351999680201212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,255,0.020608000457286835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,255,0.013882666826248169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,511,0.02589333305756251
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,511,0.0169813334941864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,2047,0.03565866748491923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,2047,0.2001439929008484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1023,0.03551999976237615
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,1023,0.02342933416366577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,4095,0.3861173391342163
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,4095,0.19127466281255087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,8191,0.7531572977701823
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1,0.008522666369875273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1,0.010559999694426855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,3,0.008618666479984919
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,8191,0.36857601006825763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,4095,0.19618133703867593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,12,4,128,1,float16,fp8,16383,0.7947573661804199
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,3,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,7,0.008693333094318708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,7,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,15,0.008714666590094566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,31,0.008576000109314919
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,15,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,63,0.008703999842206636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,31,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,63,0.01081066702802976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,127,0.008890666688481966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,255,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,8191,0.37614933649698895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,255,0.011616000284751257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,127,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,511,0.01251199965675672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,511,0.011616000284751257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1023,0.013072000195582708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,16383,1.4921120007832844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,1023,0.011567999919255575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,2047,0.011930666863918304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,4095,0.016528000434239704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,2047,0.014165333161751429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,4095,0.012202666451533636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,8191,0.02091199904680252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,16383,0.030031998952229817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,16383,0.016800000021855038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,32767,0.03311999887228012
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,32767,0.022895999252796173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,8191,0.013669333110253016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,65535,0.03826133410135905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,1,128,1,float16,fp8,65535,0.033439998825391136
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,3,0.008597333605090777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1,0.0084906667470932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,3,0.010751999914646149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,15,0.00867733359336853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,7,0.010751999914646149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,7,0.00879466657837232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,15,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,31,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,127,0.008992000172535578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,31,0.010682666053374609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,63,0.010922666639089584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,63,0.008890666688481966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,255,0.012240000069141388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,511,0.012389333297808966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,255,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1023,0.013093333691358566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,511,0.011541333049535751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,2047,0.014368000129858652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,2047,0.012042666474978128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,1023,0.011786667009194693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,4095,0.016554666062196095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,8191,0.021418665846188862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,4095,0.01250133290886879
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,8191,0.013818666338920593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,16383,0.0348693331082662
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,16383,0.017162666966517765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,32767,0.04002666721741358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,32767,0.023226665953795116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,65535,0.049653331438700356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1,0.00854399986565113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,2,128,1,float16,fp8,65535,0.03385066737731298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1,0.010847999403874079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,3,0.00879466657837232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,7,0.008816000074148178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,3,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,7,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,15,0.008639999975760778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,31,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,15,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,63,0.008810666700204214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,31,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,127,0.009056000038981438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,63,0.01081066702802976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,255,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,127,0.011055999745925268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,511,0.01258133351802826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1023,0.013258667041858038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,511,0.011690666278203329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,255,0.011813333878914515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,4095,0.017231999586025875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,2047,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,2047,0.014655999839305878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,1023,0.012037333101034164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,8191,0.026234666506449383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,8191,0.014576000471909841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,16383,0.017551999539136887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,32767,0.04098133246103922
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,4095,0.013178666432698568
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,16383,0.03123733401298523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,32767,0.02402133246262868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1,0.008682666967312494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,65535,0.20438400904337564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1,0.010816000401973724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,12,4,128,1,float16,fp8,65535,0.034815999368826546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,3,0.008634666601816813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,3,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,15,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,7,0.0086666668454806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,7,0.010549332946538925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,15,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,31,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,31,0.008693333094318708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,63,0.0107893335322539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,127,0.010741333166758219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,63,0.008767999708652496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,127,0.008992000172535578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,255,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,255,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,511,0.012400000045696894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,511,0.011711999773979187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1023,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,2047,0.014346666634082794
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,2047,0.012080000092585882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,1023,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,8191,0.021712000171343487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,4095,0.016783999900023144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,4095,0.013487999637921652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,8191,0.01651200031240781
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,16383,0.03505599995454153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,16383,0.02250666668017705
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,32767,0.04022400081157684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,32767,0.03472000112136205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,65535,0.05011733373006185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,1,128,1,float16,fp8,65535,0.05694933235645294
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1,0.010693332801262537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,3,0.012165332833925882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,7,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,15,0.01221866657336553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,3,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,7,0.010512000570694605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,31,0.01231466606259346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,63,0.01232533281048139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,31,0.010703999549150467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,63,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,127,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,255,0.011829332758982977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,255,0.014560000350077948
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,127,0.012517333030700684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,511,0.015077333897352219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,511,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,1023,0.0116799995303154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1023,0.01573866605758667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,2047,0.017024000485738117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,2047,0.012053333222866058
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,4095,0.01370666672786077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,4095,0.019861333072185516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,8191,0.028607999285062153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,8191,0.016522667060295742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,16383,0.03402133285999298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,16383,0.0229066660006841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,32767,0.04358933369318644
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,32767,0.0349386657277743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,65535,0.20151466131210327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1,0.0120319997270902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,3,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,2,128,1,float16,fp8,65535,0.057258665561676025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,7,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,3,0.010832000523805618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,15,0.012144000579913458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,7,0.010928000013033548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,15,0.01089599976936976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,31,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,63,0.010794666906197866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,127,0.012485332787036896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,31,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,127,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,63,0.012240000069141388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,255,0.014858666807413101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,255,0.011893333246310553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,511,0.011930666863918304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,511,0.015205333630243937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,1023,0.012154666086037954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,2047,0.012698666503032049
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1023,0.01591466615597407
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,2047,0.017530667285124462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,4095,0.024485332270463307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,4095,0.014271999398867289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,16383,0.039306665460268654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,8191,0.029290666182835896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,16383,0.023541333774725597
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,32767,0.03625600039958954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,32767,0.2036693294843038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,8191,0.017637333522240322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,65535,0.39311468601226807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,12,4,128,1,float16,fp8,65535,0.1920213301976522
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1,0.011770666887362799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1,0.009039999917149544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,7,0.009328000247478485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,3,0.00914666677514712
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,3,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,7,0.011711999773979187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,15,0.00919999989370505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,15,0.011786667009194693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,31,0.00919999989370505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,63,0.009461333354314169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,63,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,31,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,127,0.009818666925032934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,127,0.012128000458081564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,255,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,255,0.013781332721312841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,511,0.01874133323629697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,511,0.016666666915019352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,1023,0.022944000860055287
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1023,0.02380799998839696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,2047,0.03350933392842611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,2047,0.03475199888149897
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,8191,0.37805867195129395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,8191,0.1946773330370585
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,1,128,1,float16,fp8,4095,0.05862933397293091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1,0.011776000261306763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,4095,0.19563732544581094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,3,0.013002666334311167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,3,0.011701333026091257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,7,0.013242666920026144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,7,0.011882666498422623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,15,0.011776000261306763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,31,0.012005332857370377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,15,0.01310933381319046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,63,0.012159999459981918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,63,0.013445333888133367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,127,0.014090667168299357
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,127,0.012485332787036896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,31,0.013258667041858038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,255,0.020901332298914593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,255,0.014053333550691605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,511,0.017322666943073273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,511,0.02624000112215678
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,4095,0.37652266025543213
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,1023,0.023562667270501454
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1023,0.03563733398914337
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,4095,0.1924533247947693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,8191,0.7324159940083822
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,2047,0.19604265689849854
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1,0.016501333564519882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,2047,0.03607466568549474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,3,0.013839999834696451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1,0.013877333452304205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,7,0.01693333312869072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,2,128,1,float16,fp8,8191,0.36848000685373944
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,3,0.016645333419243496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,15,0.016895999511082966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,31,0.014101333916187286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,31,0.01724799970785777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,63,0.017445333302021027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,7,0.013882666826248169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,127,0.01836799954374631
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,63,0.014581333845853806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,127,0.01526933287580808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,15,0.014005333185195923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,255,0.017946666727463405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,2047,0.19512534141540527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,2047,0.3863893349965413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,511,0.02311466634273529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,255,0.02366400013367335
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,511,0.03327466547489166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1023,0.1995946764945984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,1023,0.033546666304270424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,4095,0.3734560012817383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1,0.008656000097592672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,4095,0.7550293604532877
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,7,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,15,0.008762666955590248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,7,0.008922666932145754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,15,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,31,0.008832000195980072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,8191,1.6750507354736328
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,31,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,63,0.010341333225369453
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,12,4,128,1,float16,fp8,8191,0.7258666356404623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,127,0.009119999905427298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,3,0.0086666668454806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,255,0.012245333443085352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,127,0.01097600037852923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,255,0.011850666254758835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,3,0.010960000256697336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,511,0.012762666990359625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1023,0.013167999684810638
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,511,0.011866666376590729
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,2047,0.01462399959564209
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,1023,0.011893333246310553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,2047,0.01350933313369751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,8191,0.02632533262173335
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,4095,0.016506666938463848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,4095,0.017605333278576534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,8191,0.022533332308133442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,16383,0.03143999973932902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,16383,0.03429866582155228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,32767,0.041162667175134025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,32767,0.058186665177345276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1,0.012053333222866058
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,65535,0.19996267557144165
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,3,0.0120319997270902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,1,128,1,float16,fp8,65535,0.10603732864061992
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,3,0.01097600037852923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,7,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,7,0.01102399950226148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,15,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,31,0.011077333241701126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,63,0.012272000312805176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,15,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,63,0.010933333386977514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,31,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,127,0.012543999900420507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,255,0.014912000546852747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,127,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,255,0.011946666985750198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,511,0.015279999623696009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,511,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,1023,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1023,0.016048000504573185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,2047,0.017685333887736004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,2047,0.013536000003417334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,4095,0.024341332415739696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,8191,0.02956799914439519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,4095,0.016757333030303318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,16383,0.03942933430274328
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,8191,0.022863999009132385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,16383,0.034917332231998444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,32767,0.19753066698710123
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,32767,0.05864533285299937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,3,0.012234666695197424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,2,128,1,float16,fp8,65535,0.1914880077044169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,65535,0.38150401910146076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1,0.011125333607196808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,3,0.011077333241701126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,7,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,15,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,7,0.011152000476916632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,31,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,15,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,31,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,63,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,127,0.012608000387748083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,255,0.014858666807413101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,127,0.011498666057984034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,511,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,255,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,2047,0.022218666970729828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1023,0.016522667060295742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,511,0.01540800059835116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,2047,0.014240000396966934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,1023,0.012565333396196365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,4095,0.027269333600997925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,8191,0.03691199918588003
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,8191,0.02345066765944163
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,4095,0.017279999951521557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,16383,0.036133334040641785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,16383,0.2017866571744283
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,32767,0.38543466726938885
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,32767,0.19119999806086221
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,65535,0.7572586536407471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,12,4,128,1,float16,fp8,65535,0.36690131823221844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1,0.012469333906968435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1,0.009472000102202097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,3,0.012426666915416718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,3,0.009685333197315535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,7,0.00960533320903778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,7,0.012400000045696894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,15,0.009578666960199675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,15,0.012554666648308435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,31,0.009653333574533463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,31,0.012506666282812754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,63,0.01009599988659223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,63,0.012768000364303589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,127,0.010469333579142889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,127,0.013125333935022354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,255,0.019141333798567455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,255,0.014858666807413101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,511,0.01794133335351944
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,511,0.024085332949956257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1023,0.03409066547950109
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,1023,0.024117333193620045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,2047,0.036544000109036766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,2047,0.1966773271560669
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1,0.016938666502634685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1,0.01443733274936676
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,3,0.01718933383623759
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,3,0.014463999619086584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,7,0.014554666976133982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,4095,0.37914133071899414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,7,0.01720533271630605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,1,128,1,float16,fp8,4095,0.1957119901974996
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,15,0.017130666722853977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,15,0.014581333845853806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,31,0.01747200017174085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,31,0.014629332969586054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,63,0.014975999792416891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,127,0.01580799991885821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,255,0.023930666347344715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,255,0.01874133323629697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,511,0.03350399931271871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,511,0.023792001108328503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,63,0.017727999637524288
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,1023,0.03435733417669932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1023,0.20652266343434653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,127,0.018863999595244724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,4095,0.37305601437886554
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,4095,0.7293279965718588
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1,0.024277334411938984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1,0.020501332978407543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,3,0.020576000213623047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,2047,0.37513601779937744
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,2,128,1,float16,fp8,2047,0.19616534312566122
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,3,0.02443733314673106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,7,0.024746666351954143
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,7,0.02051199972629547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,15,0.024874667326609295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,15,0.020501332978407543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,31,0.02517866591612498
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,63,0.02094399929046631
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,127,0.022250667214393616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,127,0.02693866689999898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,255,0.03706666578849157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,255,0.027258666853109997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,31,0.020773333807786305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,2047,0.7586186726888021
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,63,0.02552533398071925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,511,0.20172266165415445
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,511,0.037317333122094475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,2047,0.3763146797815959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1023,0.389514684677124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,1023,0.19585599501927695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,12,4,128,1,float16,fp8,4095,0.7317813237508138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,4095,1.6468532880147297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1,0.013455999394257864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,3,0.015210667004187902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1,0.015344000111023584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,3,0.013440000514189402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,15,0.013733333597580591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,15,0.015333333363135656
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,31,0.013909333695967993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,31,0.015135999768972397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,63,0.015728000551462173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,127,0.01655999943614006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,127,0.015450666348139444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,7,0.014314666390419006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,7,0.01578666642308235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,255,0.02025066688656807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,255,0.01930133377512296
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,511,0.024832000335057575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1023,0.19332265853881836
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,511,0.029733332494894665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1,0.024906667570273083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,1,128,1,float16,fp8,1023,0.03494933247566223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1,0.021290667355060577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,3,0.025386666258176167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,3,0.021381333470344543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,7,0.025797332326571148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,15,0.025546667476495106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,7,0.021354667842388153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,31,0.026015999416510265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,15,0.02128000060717265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,31,0.021130666136741638
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,63,0.026602665583292644
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,63,0.02184533327817917
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,127,0.02808533360560735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,127,0.023018665611743927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,255,0.03833599885304769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,255,0.02807466685771942
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,511,0.2149333357810974
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1,0.038986665507157646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,511,0.037845333417256675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1,0.03283733377854029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,3,0.03973866750796636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1023,0.37918933232625324
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,2,128,1,float16,fp8,1023,0.20101332664489746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,7,0.04012266546487808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,3,0.03289599965016047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,7,0.03289599965016047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,15,0.04031466692686081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,15,0.03292799989382426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,63,0.041135999063650765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,63,0.03324799984693527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,31,0.041077333192030586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,31,0.03294399877389272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,127,0.043951998154322304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,127,0.03412266572316488
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,255,0.20430932442347208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,255,0.04417066772778829
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,511,0.3954453468322754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,511,0.19894399245580038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,1,0.020794666061798733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,1,0.02257599929968516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,12,4,128,1,float16,fp8,1023,0.381658673286438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1023,0.8100852966308594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,7,0.02146133283774058
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,3,0.021381333470344543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,7,0.022656001150608063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,3,0.02266666789849599
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,15,0.021344001094500225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,31,0.02274666726589203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,31,0.02184533327817917
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,63,0.022319999833901722
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,15,0.022800001005331676
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,63,0.023157333334287006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,127,0.02399466683467229
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,127,0.024703999360402424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,255,0.030192000170548756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,1,0.04035733391841253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,255,0.034058667719364166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,1,0.03429866582155228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,511,0.19619200627009073
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,1,128,1,float16,fp8,511,0.03955733279387156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,3,0.04083733260631561
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,7,0.04154666761557261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,15,0.041434665520985924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,3,0.03449599941571554
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,31,0.03431999931732813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,7,0.034330666065216064
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,15,0.03436800092458725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,31,0.04204800228277842
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,63,0.042853335539499916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,63,0.0346666673819224
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,127,0.045370668172836304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,127,0.03526933242877325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,255,0.046069333950678505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,255,0.2193440000216166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,511,0.3885386784871419
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,1,0.0661653329928716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,1,0.0554720014333725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,3,0.06705600023269653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,2,128,1,float16,fp8,511,0.20669333140055338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,3,0.055205335219701133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,7,0.06850133339564006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,15,0.06825600067774455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,15,0.055642664432525635
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,31,0.06904533505439758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,31,0.055493334929148354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,63,0.07065066695213318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,63,0.055946667989095054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,127,0.20990933974583945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,127,0.05829333265622457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,7,0.05551999807357788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,255,0.4028746684392293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1,0.008698666468262672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,3,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,7,0.008858666444818178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,7,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,15,0.008879999940594038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,15,0.011066666493813196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,31,0.008938666433095932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,63,0.008863999818762144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,31,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,63,0.01109333336353302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,127,0.00915733352303505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,255,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,255,0.2063360015551249
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,3,0.011007999380429586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,255,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,511,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,511,0.012794667234023413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1023,0.013424000392357508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,2047,0.015082667271296183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,2047,0.016629333297411602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,4095,0.022053333620230358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,4095,0.02223466585079829
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,8191,0.027072000006834667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,8191,0.0347626656293869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,16383,0.036933332681655884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,16383,0.05815466741720835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,32767,0.19684267044067383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,32767,0.10625599821408589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,1023,0.013594667116800943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1,0.012069333344697952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,1,128,1,float16,fp8,65535,0.21292267243067423
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,3,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,65535,0.382479985555013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,3,0.011087999989589056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,7,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,7,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,15,0.012202666451533636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,31,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,31,0.011039999624093374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,15,0.011152000476916632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,12,4,128,1,float16,fp8,511,0.39115198453267414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,63,0.012421333541472753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,511,0.8023680051167806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,63,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,127,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,255,0.014794666320085526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,127,0.011407999942700068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,511,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1023,0.01664000004529953
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,1023,0.013557333499193192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,2047,0.022330666581789654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,4095,0.027210667729377747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,2047,0.016789333273967106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,4095,0.022789334257443745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,8191,0.03454933315515518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,8191,0.03709866603215536
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,16383,0.058634668588638306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,16383,0.19646400213241577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,32767,0.37646933396657306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,32767,0.19218667348225912
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,65535,0.7369600137074789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1,0.011338666081428528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,65535,0.37041600545247394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,3,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,3,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,7,0.011322667201360067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,7,0.01249066616098086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,15,0.012383999923865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,15,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,31,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,63,0.012448000411192576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,63,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,31,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,127,0.012671999633312225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,2,128,1,float16,fp8,255,0.012037333101034164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,127,0.011535999675591787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,255,0.015125333021084467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,255,0.012442667037248611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,511,0.01589866727590561
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,511,0.01545599972208341
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,511,0.012533333152532578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1023,0.021125334004561108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,2047,0.026447998980681103
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,1023,0.014106666048367819
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,4095,0.0403466671705246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,2047,0.017375999440749485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,4095,0.023418667415777843
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,8191,0.20066134134928384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,8191,0.03594133257865906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,16383,0.19158399105072021
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,16383,0.3863626718521118
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,32767,0.8369279702504476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,65535,0.7225333054860433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,12,4,128,1,float16,fp8,32767,0.4049066702524821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,65535,1.6687307357788086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,1,0.030720000465710957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,1,0.03602666656176249
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,3,0.03642666588226954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,3,0.030810666580994923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,7,0.03699733316898346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,15,0.036933332681655884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,7,0.03090133269627889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,15,0.03068800022204717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,31,0.037530665596326195
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,31,0.030879999200503033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,63,0.03841600070397059
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,63,0.031045332551002502
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,127,0.032287999987602234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,127,0.0402453343073527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,1,0.0687253326177597
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,255,0.223578671614329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,1,0.050954664746920265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,1,128,1,float16,fp8,255,0.04354133208592733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,3,0.07008533179759979
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,3,0.05100800096988678
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,7,0.070933332045873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,7,0.05082666873931885
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,15,0.051034669081370033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,15,0.07116800049940745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,31,0.05113600194454193
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,31,0.07194666564464569
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,63,0.0740586668252945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,63,0.05187733471393585
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,127,0.21146667003631592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,1,0.12223466237386067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,127,0.05484800040721893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,1,0.09204799930254619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,255,0.40140267213185626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,2,128,1,float16,fp8,255,0.20572799444198608
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,3,0.12361066540082295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,3,0.09177066882451375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,7,0.12585066755612692
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,15,0.12573333581288657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,7,0.092031995455424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,31,0.09241066376368205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,31,0.12735999623934427
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,15,0.09221866726875305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,63,0.21985065937042236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,63,0.09373333056767781
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,127,0.41887466112772626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,127,0.22445333003997803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,255,0.8181173006693522
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,12,4,128,1,float16,fp8,255,0.4198133150736491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,1,0.06303999821345012
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,1,0.052154665191968284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,3,0.06429333488146464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,3,0.0525439977645874
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,7,0.06523733337720235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,7,0.05260799825191498
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,15,0.0651039977868398
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,15,0.05259733398755392
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,31,0.066021333138148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,31,0.05306133131186167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,63,0.06822933256626129
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,63,0.05354666709899902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,127,0.23812266190846762
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,1,128,1,float16,fp8,127,0.05612266560395559
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,1,0.12653866410255432
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,1,0.09193600217501323
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,3,0.09258666634559631
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,3,0.12902399897575378
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,7,0.130949338277181
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,7,0.092357337474823
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,15,0.13061333696047464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,15,0.0925546685854594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,31,0.1371893286705017
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,31,0.09315733114878337
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,63,0.23384533325831094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,63,0.09461866815884908
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,127,0.46563732624053955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,2,128,1,float16,fp8,127,0.21902932723363241
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,1,0.23317867517471313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,1,0.17707200845082602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,3,0.23786133527755737
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,3,0.17758933703104654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,7,0.2411253253618876
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,7,0.17742933829625449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,15,0.2490560015042623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,15,0.1775839924812317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,31,0.28938132524490356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1,0.008752000207702318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,3,0.008821333448092142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1,0.011215999722480774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,3,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,7,0.00890666681031386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,7,0.011237333218256632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,15,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,15,0.008832000195980072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,31,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,31,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,63,0.00901333304742972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,63,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,127,0.009216000015536943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,127,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,255,0.012538666526476542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,255,0.012047999848922094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,511,0.01303999995191892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,511,0.013429333766301474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1023,0.014256000518798828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,1023,0.016437333077192307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,2047,0.02015999952952067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,2047,0.02236266682545344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,4095,0.02518400053183238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,31,0.17918932437896729
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,4095,0.03438399980465571
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,8191,0.03505599995454153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,8191,0.05818133552869161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,16383,0.10619200269381206
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,16383,0.1958720088005066
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,32767,0.4168533484141032
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,1,128,1,float16,fp8,32767,0.2142933408419291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1,0.012410666793584824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,3,0.012421333541472753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1,0.011029332876205444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,3,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,7,0.012442667037248611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,7,0.011120000233252844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,15,0.01249066616098086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,15,0.011215999722480774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,31,0.011237333218256632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,31,0.012576000144084295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,63,0.012634667257467905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,63,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,127,0.012810666114091873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,127,0.011509332805871964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,63,0.4501333236694336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,255,0.01534933348496755
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,255,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,511,0.01613333324591319
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,63,0.23009065786997476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,511,0.013829333086808523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,1023,0.016613333175579708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,2047,0.02664533257484436
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,2047,0.022986667851607006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,4095,0.03473066786924998
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,8191,0.1964906652768453
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,8191,0.05892266829808553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,16383,0.37827734152475995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,16383,0.19221333662668863
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1023,0.021664001047611237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,4095,0.04079466561476389
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,2,128,1,float16,fp8,32767,0.3710079987843831
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,32767,0.732917308807373
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1,0.01137599969903628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1,0.012618667135636011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,3,0.012682666381200155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,3,0.011418666690587997
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,7,0.012703999876976013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,7,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,15,0.012752000242471695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,31,0.012837332983811697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,15,0.011381333072980246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,31,0.011407999942700068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,63,0.01292266696691513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,63,0.011381333072980246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,127,0.013338666409254074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,127,0.01181866725285848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,255,0.01590399940808614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,511,0.020975999534130096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,255,0.012794667234023413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,511,0.014463999619086584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1023,0.02609066665172577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,2047,0.03554133325815201
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,1023,0.01735466718673706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,2047,0.023792001108328503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,4095,0.036117332677046456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,4095,0.20069867372512817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,12,4,128,1,float16,fp8,127,0.4225813150405884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,8191,0.38579734166463214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,8191,0.19139732917149863
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,16383,0.367631991704305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,16383,0.7550186316172282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,3,0.008799999952316284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,12,4,128,1,float16,fp8,32767,0.7950666745503744
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,32767,1.6727892557779949
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,7,0.009066666786869368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,7,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,3,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,15,0.008954666554927826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,31,0.009008000294367472
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,15,0.011109333485364914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,31,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,63,0.009018666421373686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,127,0.009423999736706415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,63,0.01126933346192042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,127,0.011429333438475927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,255,0.012853333105643591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,255,0.01292266696691513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1,0.008874666566650072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,511,0.013701333353916803
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,511,0.015967999895413715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1023,0.018976000448067982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,1023,0.021984001000722248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,2047,0.023797333240509033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,2047,0.033914667864640556
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,8191,0.10546132922172546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,8191,0.19416000445683798
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,4095,0.03809600075085958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,4095,0.05774400134881338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,16383,0.3765386740366618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,32767,0.41276800632476807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,1,128,1,float16,fp8,16383,0.2129759987195333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1,0.012565333396196365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,32767,0.7332692941029867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1,0.011285333583752314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,127,0.907152016957601
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,3,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,3,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,7,0.011301333705584208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,15,0.012618667135636011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,15,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,7,0.012703999876976013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,31,0.012784000486135483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,63,0.012928000340859095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,63,0.01137599969903628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,127,0.011754666765530905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,127,0.01313599944114685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,255,0.01573866605758667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,255,0.013104000439246496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,511,0.020831999679406483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,511,0.01628799984852473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,1023,0.022469334304332733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1023,0.0260959987839063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,4095,0.058373332023620605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,4095,0.19501866896947226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,2047,0.034160000582536064
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,16383,0.3705120086669922
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,2047,0.0356480007370313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,8191,0.19125332434972128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,8191,0.3746773401896159
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,16383,0.809397300084432
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,3,0.011429333438475927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,3,0.012778667112191519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1,0.011488000551859537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,32767,0.8037280241648356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1,0.012719999998807907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,7,0.013034666577974955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,7,0.011525332927703857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,15,0.011349332829316458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,15,0.01292266696691513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,32767,1.444053332010905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,31,0.011493333925803503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,31,0.013199999928474426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,63,0.013167999684810638
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,63,0.01157333329319954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,127,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,2,128,1,float16,fp8,31,0.011359999577204386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,127,0.01368533323208491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,255,0.020645332833131153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,255,0.01379199946920077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,511,0.025616000096003216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,511,0.016842667013406754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,2047,0.19991999864578247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1023,0.03544000039498011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,2047,0.035546667873859406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,1023,0.02293866624434789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,4095,0.19099199771881104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,8191,0.36717331409454346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,8191,0.7516213258107504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,16383,0.7940320173899332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1,0.01157333329319954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1,0.013445333888133367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,3,0.011674666156371435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,3,0.01370666672786077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,7,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,4,128,1,float16,fp8,32767,1.5736799240112305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,16383,1.4862559636433919
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,7,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,15,0.013658666362365087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,31,0.011946666985750198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,15,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,63,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,63,0.014074667046467463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,127,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,31,0.013557333499193192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,127,0.014965333044528961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,255,0.01806933308641116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,32767,2.9546613693237305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,255,0.017786666750907898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,511,0.0229120006163915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,511,0.028010666370391846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1023,0.21278933684031168
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,1023,0.033088001112143196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,2047,0.37677331765492755
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,4095,0.7440640131632487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,4095,0.37194132804870605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,2047,0.20685333013534546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,8191,0.7308106422424316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1,0.010527999450763067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1,0.008442666381597519
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,8191,1.481119950612386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,16383,1.5451359748840332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,3,0.008570666735370954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,3,0.01051733394463857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,7,0.008581333483258883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,7,0.010693332801262537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,15,0.008576000109314919
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,15,0.010543999572594961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,16383,3.3211841583251953
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,31,0.010634666929642359
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,63,0.010469333579142889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,63,0.00877333308259646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,31,0.00860799973209699
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,255,0.012047999848922094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,127,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,127,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,511,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,255,0.01145600030819575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,511,0.011461333682139715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1023,0.013088000317414602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,1023,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,2047,0.013989333063364029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,2047,0.011578666667143503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,4095,0.016330666840076447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,8191,0.02072000006834666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,8191,0.01331199953953425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,4095,0.01202133297920227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,16383,0.01648533344268799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,16383,0.030074665943781536
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,8,8,128,1,float16,fp8,32767,3.0816160837809243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,32767,0.032074667513370514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,65535,0.03310933212439219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,65535,0.037802666425704956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,131071,0.04797866443792979
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,32767,0.02258133391539256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1,0.008522666369875273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1,0.010586666564146677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,3,0.008757333581646284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,3,0.010645333677530289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,7,0.008602666358153025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,7,0.010661333799362183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,15,0.00855466661353906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,63,0.008736000085870424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,15,0.010687999427318573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,31,0.010575999816258749
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,63,0.010597333312034607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,31,0.008736000085870424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,1,128,1,float16,fp8,131071,0.05555200080076853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,127,0.008869333192706108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,255,0.012080000092585882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,127,0.010640000303586325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,255,0.01139733319481214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,511,0.011509332805871964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,32767,6.657434463500977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1023,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,511,0.01249066616098086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,1023,0.011519999553759893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,2047,0.01434133326013883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,2047,0.011770666887362799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,4095,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,8191,0.021407999098300934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,8191,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,16383,0.03472000112136205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,16383,0.01682666689157486
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,32767,0.03962666789690653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,4095,0.016447999825080235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,32767,0.023130667706330616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,65535,0.04901333153247833
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,65535,0.03350399931271871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,2,128,1,float16,fp8,131071,0.05603733162085215
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,131071,0.20438400904337564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,3,0.00860799973209699
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1,0.008703999842206636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1,0.010677333921194077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,3,0.010672000547250112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,7,0.00871999996403853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,7,0.010597333312034607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,15,0.008661333471536636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,31,0.008752000207702318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,63,0.009061333412925402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,31,0.010618666807810465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,127,0.00897066667675972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,15,0.010682666053374609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,63,0.010591999938090643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,255,0.012154666086037954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,255,0.011525332927703857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,511,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,127,0.010832000523805618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1023,0.013130666067202887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,1023,0.011823999385039011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,511,0.011584000041087469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,2047,0.011957333733638128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,2047,0.014607999473810196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,4095,0.012800000607967377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,4095,0.01718933383623759
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,8191,0.02589866767326991
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,16383,0.030565333863099415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,32767,0.04044266790151596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,16383,0.017370666066805523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,8191,0.014282666146755219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,65535,0.2042400042215983
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,32767,0.024127999941507976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,65535,0.034688000877698265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1,0.0075093333919843035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,4,128,1,float16,fp8,131071,0.19136534134546915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,131071,0.397269328435262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,3,0.007749333356817563
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,15,0.007711999739209811
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,3,0.011221333096424738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,7,0.007760000104705493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,15,0.010901333143313726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,7,0.010960000256697336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,31,0.007946666950980822
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,31,0.01184533288081487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,127,0.008053333188096682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,63,0.007776000226537387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,511,0.01492799942692121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,255,0.010410666465759277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,127,0.011242666592200598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,255,0.012634667257467905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,2047,0.0429013321797053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1023,0.024282666544119518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,511,0.013690666606028875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,4095,0.08036266764005025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,1023,0.014096000542243322
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,2047,0.014538666854302088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,8191,0.15492266416549683
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,4095,0.015487999965747198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,8191,0.018239999810854595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,16383,0.30429333448410034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,32767,0.6126826604207357
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,16383,0.02350933353106181
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,65535,0.1921493411064148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,32767,0.03397866586844126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,65535,1.217146635055542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,3,0.010591999938090643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1,0.00854399986565113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,8,8,128,1,float16,fp8,131071,0.3710879882176717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1,0.010538666198650995
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,3,0.008661333471536636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,7,0.008757333581646284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,15,0.010527999450763067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,15,0.008650666723648706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,31,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,31,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,131071,2.429504076639811
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,63,0.010645333677530289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,255,0.012165332833925882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,63,0.00873066671192646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,127,0.009088000282645226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,127,0.010847999403874079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,511,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,511,0.012613333761692047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,2047,0.014309333016475042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,255,0.011535999675591787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,1023,0.011567999919255575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1023,0.012981332838535309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,2047,0.011968000481526056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,4095,0.016714667280515034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,4095,0.01332266628742218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,8191,0.016384000579516094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,8191,0.02142400046189626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,16383,0.034847999612490334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,16383,0.02252800017595291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,32767,0.04015466570854187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,32767,0.03433600068092346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,65535,0.04976533353328705
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1,0.012063999970753988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,131071,0.20708266894022623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,65535,0.0569706658522288
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,1,128,1,float16,fp8,131071,0.10171733299891154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1,0.010629333555698395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,7,0.012069333344697952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,15,0.012122667084137598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,3,0.010757333288590113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,7,0.010672000547250112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,3,0.011893333246310553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,15,0.010661333799362183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,31,0.012122667084137598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,63,0.012223999947309494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,31,0.010794666906197866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,127,0.01240533341964086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,63,0.010784000158309937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,127,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,255,0.014698666830857595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,255,0.011711999773979187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,511,0.015119999647140503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1023,0.015664000064134598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,511,0.011594666788975397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,1023,0.011776000261306763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,2047,0.016970666746298473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,4095,0.01974933346112569
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,8191,0.028490667541821797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,2047,0.01201066623131434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,4095,0.013717333475748697
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,8191,0.016693333784739178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,16383,0.03370666752258936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,16383,0.022709332406520844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,32767,0.04341333111127218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,32767,0.03510933369398117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,65535,0.057258665561676025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,65535,0.20112532377243042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,131071,0.387935996055603
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,3,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,2,128,1,float16,fp8,131071,0.1931040088335673
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,3,0.010837333897749582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1,0.012037333101034164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,7,0.012085333466529846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,15,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,7,0.01071999967098236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,15,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,31,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,63,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,31,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,127,0.01257066677014033
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,63,0.01081066702802976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,255,0.014858666807413101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,127,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,255,0.011685332904259363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,511,0.015146666516860327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,2047,0.017583999782800674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1023,0.01613333324591319
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,511,0.011877333124478659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,1023,0.011973333855470022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,2047,0.012448000411192576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,4095,0.024282666544119518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,4095,0.41102401415507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,8191,0.029482667644818623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,4095,0.014240000396966934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,8191,0.01738133281469345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,16383,0.03923733284076055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,16383,0.023391999304294586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,32767,0.20337599515914917
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1,0.0075519997626543045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,32767,0.036277333895365395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,65535,0.19139200448989868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1,0.011055999745925268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,4,128,1,float16,fp8,131071,0.368831992149353
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,3,0.00772266648709774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,3,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,131071,0.7604320049285889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,7,0.007776000226537387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,65535,0.39265600840250653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,31,0.007823999971151352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,15,0.010960000256697336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,15,0.007797333101431529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,63,0.007936000203092894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,127,0.00810666692753633
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,31,0.011087999989589056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,7,0.011450666934251785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,127,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,255,0.010437333335479101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,511,0.014970666418472925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,255,0.013983999689420065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,511,0.014080000420411428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,1023,0.014314666390419006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,2047,0.015466666469971338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,4095,0.08040533463160197
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,2047,0.043023998538653054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1023,0.02454400062561035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,8191,0.15528000394503275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,4095,0.01812800019979477
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,8191,0.02314666658639908
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,16383,0.03342933456103007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,16383,0.3109333316485087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,32767,0.19395200411478677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,32767,0.6158560117085775
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,65535,1.2240479787190754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,65535,0.37069865067799884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,131071,0.7246932983398438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1,0.008901333436369896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,3,0.011525332927703857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,3,0.00921066664159298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,131071,2.4367626508076987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,7,0.009183999771873156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,7,0.011450666934251785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,15,0.009226666763424873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,31,0.009312000125646591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,15,0.011594666788975397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,31,0.011482667177915573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,63,0.009349333122372627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,63,0.012869333227475485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,127,0.009712000067035357
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,255,0.013663999736309052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,127,0.011978667229413986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,511,0.01852799952030182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,511,0.0163680004576842
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1023,0.023455999791622162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,255,0.013674666484196981
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,8191,0.19352533419926962
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,2047,0.033200000723203026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,1023,0.022730665902296703
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,2047,0.03454400102297465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,8191,0.3763466676076253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,4095,0.05824000140031179
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,4095,0.19428267081578574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,3,0.012896000097195307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,16383,0.7330773671468099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1,0.012853333105643591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1,0.011760000139474869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,1,128,1,float16,fp8,16383,0.3712213436762492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,15,0.011754666765530905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,7,0.011711999773979187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,31,0.01303999995191892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,15,0.013114667187134424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,31,0.011685332904259363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,3,0.011690666278203329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,7,0.013072000195582708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,63,0.013349333157142004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,63,0.011999999483426413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,127,0.013999999811251959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,127,0.012389333297808966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,255,0.020997333029905956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,255,0.01392000044385592
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,511,0.0259253333012263
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,511,0.01716800034046173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,4095,0.37481598059336346
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1023,0.03562666724125544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,1023,0.02327999969323476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,8,8,128,1,float16,fp8,63,0.011509332805871964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,4095,0.19150932629903158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,2047,0.19568000237147012
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,2047,0.03588266670703888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,8191,0.36762134234110516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1,0.01369599997997284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1,0.016735999534527462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,8191,0.729914665222168
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,7,0.016864000509182613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,7,0.01368533323208491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,15,0.016917333006858826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,2,128,1,float16,fp8,16383,0.7232853571573893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,15,0.013882666826248169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,3,0.01687466725707054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,3,0.013861333330472311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,16383,1.4415465990702312
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,31,0.017024000485738117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,31,0.0138026662170887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,63,0.01735466718673706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,63,0.014389333625634512
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,127,0.0185759998857975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,127,0.015082667271296183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,255,0.023381332556406658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,255,0.018085333208243053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,511,0.03289066751797994
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,511,0.022853332261244457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,2047,0.3859786589940389
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1023,0.19950399796168009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,2047,0.19460266828536987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,1023,0.03332266708215078
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,4095,0.7555039723714193
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,8191,0.7271733283996582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,4095,0.3719093402226766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1,0.019861333072185516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,3,0.01903466631968816
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,8191,1.491594632466634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1,0.018677332748969395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,3,0.01977066695690155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,7,0.01918399954835574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,7,0.019904000063737232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,15,0.019248000035683315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,4,128,1,float16,fp8,16383,1.5452799797058105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,15,0.019925333559513092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,31,0.019461333751678467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,31,0.019999999552965164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,63,0.019823999454577763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,63,0.020207999895016353
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,127,0.021722666919231415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1023,0.3789279858271281
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,255,0.031141333281993866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,127,0.02147199958562851
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,255,0.026634665826956432
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,1023,0.19399466117223105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,16383,2.968448003133138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,511,0.19326933224995932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,2047,0.7461547056833903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,511,0.03679466744263967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,2047,0.3731573422749837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,3,0.008672000219424566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,4095,0.7278986771901449
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,3,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,4095,1.484005292256673
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,7,0.008618666479984919
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,8191,1.4357919692993164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,7,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,31,0.008869333192706108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,15,0.00878399983048439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,15,0.010725333044926325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,8191,3.3293546040852866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,63,0.008821333448092142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,63,0.010863999525705973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,127,0.01099733387430509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,255,0.011642667154471079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,127,0.00902399979531765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,511,0.012730666746695837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,255,0.0124746672809124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1023,0.01320533330241839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,511,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,1023,0.011834666132926941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,2047,0.014730667074521383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,4095,0.017237332959969837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,2047,0.013424000392357508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,8,8,128,1,float16,fp8,16383,3.082085291544596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,31,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,8191,0.022431999444961548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,4095,0.01647466669480006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,16383,0.031258667508761086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,8191,0.02624000112215678
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,32767,0.058117335041364036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,16383,0.03436800092458725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,65535,0.19964265823364258
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,65535,0.10578667124112447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,1,128,1,float16,fp8,131071,0.2132320006688436
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,131071,0.3874719937642415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,3,0.012117333710193634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,32767,0.04094400008519491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1,0.011994666109482447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,3,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,7,0.012122667084137598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,15,0.012096000214417776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,31,0.012362666428089142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,63,0.012272000312805176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,15,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,31,0.010842667271693548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,63,0.010794666906197866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,127,0.012437333663304647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,127,0.011194666226704916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,511,0.01516266663869222
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,255,0.011695999652147293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1023,0.015840000162522
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,511,0.011893333246310553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,1023,0.011989332735538483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,255,0.014869333555301031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,4095,0.024218666056791942
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,8191,0.029487999776999157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,4095,0.01655999943614006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,16383,0.03935466706752777
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,2047,0.017738666385412216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,8191,0.02260799954334895
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,2047,0.013530666629473368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,16383,0.034671999514102936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,32767,0.19710934162139893
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,32767,0.05866133173306783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,65535,0.1923733353614807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,65535,0.3802613417307536
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1,0.012080000092585882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1,0.011061333119869232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,2,128,1,float16,fp8,131071,0.37037332852681476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,7,0.012202666451533636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,3,0.012128000458081564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,3,0.01099733387430509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,131071,0.827573299407959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,7,0.011039999624093374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,31,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,31,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,15,0.012170666207869848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,63,0.01109333336353302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,63,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,127,0.012495999534924826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,127,0.011359999577204386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,255,0.01504533365368843
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,511,0.01544533297419548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1023,0.01647466669480006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,255,0.011978667229413986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,2047,0.022218666970729828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,511,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,1023,0.012549333274364471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,4095,0.026965332527955372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,2047,0.014197333405415217
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,8191,0.03700799991687139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,4095,0.01710933322707812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,16383,0.20088533560434976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,8191,0.023370665808518726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,16383,5.900447845458984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,16383,0.03577066709597906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,32767,0.19142399231592813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,32767,0.3853653271993001
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1,0.00761600024998188
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,65535,0.36683201789855957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1,0.012853333105643591
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,65535,0.8387573560078939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,3,0.007760000104705493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,7,0.00784533346692721
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,3,0.012991999586423239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,7,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,15,0.007967999825874964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,131071,1.4931093851725261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,15,0.012975999464591345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,31,0.007903999959429106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,31,0.012901333471139273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,63,0.008047999814152718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,63,0.012906666845083237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,127,0.008133333176374435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,127,0.013461332768201828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,255,0.010399999717871347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,255,0.014245333770910898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,511,0.015274666249752045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1023,0.024506665766239166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,511,0.014538666854302088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,1023,0.015333333363135656
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,2047,0.04310933252175649
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,2047,0.017973333597183228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,4095,0.023247999449570973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,4,128,1,float16,fp8,131071,0.7221013704935709
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,8191,0.18729066848754883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,8191,0.033471999069054924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,4095,0.08072533210118611
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,16383,0.19286932547887167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,32767,0.7188800175984701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,32767,0.37056533495585126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,16383,0.4069226582845052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,65535,0.7241919835408529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,65535,1.4312639236450195
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1,0.009306666751702627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,8,8,128,1,float16,fp8,131071,1.545738697052002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,3,0.009392000113924345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,3,0.012063999970753988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,7,0.009488000224033991
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,131071,3.2081918716430664
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,7,0.012042666474978128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,15,0.009413333609700203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,15,0.011909333368142446
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,31,0.012058666596810022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,31,0.009695999945203463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,63,0.009808000177145004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,63,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,127,0.010602666685978571
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,255,0.01858666663368543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,127,0.012938667088747025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,255,0.014277332772811254
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,511,0.02348800003528595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1023,0.03332799921433131
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,511,0.017477333545684814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,1023,0.023754666248957317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,2047,0.03598399957021078
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,8191,0.7335626284281412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1,0.016730666160583496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,8191,0.3715039889017741
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,2047,0.19475199778874716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,3,0.01672533278663953
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,1,128,1,float16,fp8,4095,0.19436800479888916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1,0.014202666779359182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,3,0.01422400027513504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,4095,0.37831465403238934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,7,0.016895999511082966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,7,0.014111999422311783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,15,0.016832000265518825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,31,0.01709866647919019
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,31,0.01434133326013883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,63,0.01461333284775416
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,127,0.018543999642133713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,127,0.01552533358335495
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,255,0.023386667172114056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,255,0.018426666657129925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,511,0.03311466674009959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,15,0.014042666802803675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,63,0.017386666188637417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,511,0.023541333774725597
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,4095,0.3732159932454427
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,1023,0.03356799980004629
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1023,0.19315733512242636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,4095,0.7285813490549723
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1,0.02370133250951767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,8191,1.5520213445027669
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,8191,0.7264800071716309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,2047,0.37301866213480633
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,3,0.020207999895016353
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,2,128,1,float16,fp8,2047,0.20809600750605264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1,0.020128000527620316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,3,0.024058667321999867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,7,0.024271999796231587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,7,0.020266667008399963
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,15,0.024453334510326385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,31,0.02470933397610982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,63,0.02498133232196172
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,63,0.020634666085243225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,127,0.026560001075267792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,127,0.021925332645575207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,255,0.036330667634805046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,255,0.026890667776266735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,15,0.020309332758188248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,31,0.020309332758188248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,511,0.2007253368695577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,511,0.03692800054947535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,2047,0.37510931491851807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,2047,0.7565706570943197
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,4095,1.664453347524007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,4095,0.7867999871571859
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,1023,0.19523199399312338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1,0.03251733382542928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1023,0.3877760171890259
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1,0.031701333820819855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,3,0.03329599897066752
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,7,0.031557333966096245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,15,0.03178133318821589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,3,0.031744000812371574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,4,128,1,float16,fp8,8191,1.438154697418213
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,31,0.034330666065216064
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,31,0.03163733333349228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,63,0.03676799933115641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,8191,2.965818723042806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,7,0.03370666752258936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,63,0.03178133318821589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,15,0.03350399931271871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,127,0.03275199979543686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,127,0.03695466617743174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,255,0.19492799043655396
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,1023,0.37534932295481366
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,255,0.04306666553020477
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1023,0.751360019048055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,2047,0.7311200300852457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,2047,1.4910292625427246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,511,0.42005332310994464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,511,0.19446400801340738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,4095,1.442303975423177
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1,0.013114667187134424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1,0.014933332800865173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,4095,2.972640037536621
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,7,0.013568000247081121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,3,0.013424000392357508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,8,8,128,1,float16,fp8,8191,3.0865227381388345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,3,0.014725333700577417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,7,0.014698666830857595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,15,0.013669333110253016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,15,0.014682666709025701
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,31,0.013621332744757334
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,31,0.01471466695268949
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,63,0.015365333606799444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,127,0.015279999623696009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,127,0.016234666109085083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,8191,5.935946782430013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,255,0.020197333147128422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,255,0.01882133384545644
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,511,0.0240639994541804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,511,0.029738667110602062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1023,0.1914880077044169
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1,0.024346667031447094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,1023,0.034416000048319496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1,0.020687999824682873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,3,0.020714666694402695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,3,0.02483733246723811
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,1,128,1,float16,fp8,2047,0.197050670782725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,7,0.025248001019159954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,7,0.020741333564122517
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,15,0.025077333052953083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,15,0.02077866718173027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,2047,0.413642684618632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,31,0.025536000728607178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,31,0.020970667401949566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,63,0.02587199956178665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,63,0.021146667500336964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,127,0.027615999182065327
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,255,0.037402667105197906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,255,0.02794133375088374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,127,0.022645334402720135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,511,0.19556800524393717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,511,0.03693866729736328
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1023,0.3760853211085002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1,0.03884266565243403
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,1023,0.19841599464416504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1,0.03258133431275686
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,3,0.03912533322970072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,7,0.03980266551176707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,3,0.0323786661028862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,2047,0.7314399878184
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,2,128,1,float16,fp8,2047,0.40564799308776855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,7,0.032357332607110344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,15,0.03957333415746689
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,15,0.03257599969704946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,31,0.040448000033696495
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,31,0.032458665470282234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,127,0.03320533285538355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,127,0.043280000487963356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,255,0.2028533418973287
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,255,0.044064000248909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,63,0.041002665956815086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,511,0.39213867982228595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,63,0.03287466615438461
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,511,0.19707733392715454
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,2047,1.5057226816813152
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,2047,0.7350666522979736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,3,0.05180799961090088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,4,128,1,float16,fp8,1023,0.37910401821136475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1,0.051957334081331887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1023,0.8076372941335043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,7,0.05975999931494395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,3,0.058693334460258484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1,0.0577706644932429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,7,0.05189333359400431
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,15,0.05192000170548757
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,31,0.05197333296140035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,63,0.06131199995676676
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,63,0.05230399966239929
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,127,0.19609065850575766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,15,0.05983999868233999
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,127,0.053946668903032936
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,255,0.19790399074554443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,255,0.38364267349243164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,31,0.06091733276844025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1023,1.4890987078348796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,1023,0.7355999946594238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,511,0.38039998213450116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,511,0.8306612968444824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1,0.020586666961510975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1,0.021733333667119343
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,8,8,128,1,float16,fp8,2047,1.4468213717142742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,2047,2.9609813690185547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,3,0.021018666525681812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,7,0.02189333240191142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,15,0.021295999487241108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,15,0.021930667261282604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,31,0.02143466720978419
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,31,0.021882665654023487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,3,0.02183466653029124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,7,0.02124800036350886
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,63,0.021882665654023487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,63,0.022304000953833263
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,127,0.02370133250951767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,127,0.023333333432674408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,255,0.03350399931271871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,255,0.029103999336560566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,511,0.19433067242304483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,511,0.0383146678407987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1,0.03934400031963984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1023,0.37539732456207275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,1,128,1,float16,fp8,1023,0.20165866613388062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1,0.033488000432650246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,3,0.040192000567913055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,3,0.03330666571855545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,7,0.04068266600370407
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,7,0.03324799984693527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,31,0.041264000038305916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,31,0.03358400116364161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,15,0.040906667709350586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,15,0.03345066557327906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,63,0.04192533095677694
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,63,0.03369066615899404
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,127,0.044122666120529175
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,127,0.034346667428811394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,255,0.1985973318417867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,255,0.044997334480285645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,511,0.38413333892822266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,511,0.20305599768956503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,3,0.06674666702747345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1023,0.740885337193807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1,0.06572799881299336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1,0.05438933273156484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,2,128,1,float16,fp8,1023,0.3840213219324748
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,3,0.05429866909980774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,7,0.0673280010620753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,7,0.05451733370621999
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,31,0.06846933563550313
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,15,0.0547680010398229
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,63,0.06942399839560191
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,63,0.05500266452630361
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,15,0.0676746666431427
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,31,0.05483733117580414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,127,0.20691200097401938
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,127,0.05659199754397074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,255,0.3982986609141032
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,255,0.20311466852823892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1,0.10963199536005656
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,511,0.7707413037618002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,511,0.3850719928741455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1,0.09378133217493693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,3,0.09356266260147095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,3,0.11152533690134685
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,7,0.11348266402880351
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,4,128,1,float16,fp8,1023,0.7918346722920736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,7,0.09345600008964539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,15,0.11372266213099162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,15,0.09382399916648865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1023,1.6773920059204102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,31,0.11452266573905945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,31,0.09407466650009155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,63,0.0946613351504008
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,127,0.3893866539001465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,127,0.20500266551971436
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,255,0.3868480126063029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,255,0.7591893672943115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,63,0.1993173360824585
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,1023,1.455626646677653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1,0.008693333094318708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,3,0.010863999525705973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1023,2.968282699584961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,7,0.00891733355820179
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,3,0.008863999818762144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,7,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,31,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,15,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,31,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,63,0.00892800030608972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,63,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,127,0.009136000027259191
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,255,0.011839999506870905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,127,0.011130666981140772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,255,0.012250666817029318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,511,0.012586666891972223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1,0.011018666128317514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,511,0.011882666498422623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,1023,0.01320533330241839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1023,0.013466666142145792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,2047,0.015226667126019796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,2047,0.016554666062196095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,8191,0.02714666724205017
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,4095,0.02201066662867864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,4095,0.022202665607134502
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,8191,0.03440000116825104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,16383,0.036687999963760376
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,16383,0.05825066566467285
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,32767,0.10575999816258748
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,32767,0.19664533933003744
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,65535,0.3808106581370036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,65535,0.21380800008773804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,3,0.012293333808581034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,131071,0.7384373346964518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,1,128,1,float16,fp8,131071,0.4140799840291341
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,3,0.010938666760921478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,7,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,15,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,7,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,15,0.010938666760921478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,31,0.012410666793584824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,63,0.012586666891972223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,31,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,63,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,127,0.012613333761692047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,127,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,255,0.015024000157912573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,255,0.01198400060335795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,511,0.015311999867359797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,511,0.0120319997270902
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1023,0.016656000167131424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,1023,0.013776000589132309
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,2047,0.022175999979178112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,4095,0.02752533306678136
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,2047,0.016597333053747814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,511,1.6707199414571126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,8191,0.0369759996732076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,4095,0.02274133265018463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,8191,0.03480000048875809
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,16383,0.19610132773717245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,16383,0.058602665861447654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,32767,0.3757226864496867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,65535,0.369701345761617
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,65535,0.7328106562296549
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,8,8,128,1,float16,fp8,511,0.7440000375111898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1,0.01138666644692421
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,131071,0.7280800342559814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,7,0.012400000045696894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,3,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,131071,1.6225013732910156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,15,0.011157333850860596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,7,0.011274666835864386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,15,0.012522666404644648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,31,0.01109333336353302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,31,0.012367999802033106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,63,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,127,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,63,0.01239466667175293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,127,0.011429333438475927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,255,0.01523200049996376
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,255,0.012154666086037954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,511,0.012586666891972223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1023,0.021221332252025604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,511,0.01609066625436147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,1023,0.014032000054915747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,2047,0.026261332134405773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,4095,0.040133332212766014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,2047,0.01735466718673706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,4095,0.023402666052182514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,8191,0.19984533389409384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,8191,0.03606399893760681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,16383,0.3859573205312093
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,32767,0.36668264865875244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,32767,0.7532052993774414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,16383,0.2056586742401123
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,65535,0.7209386825561523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,65535,1.670346736907959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1,0.013088000317414602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1,0.00784533346692721
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,3,0.007978666573762894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,3,0.01320533330241839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,4,128,1,float16,fp8,131071,1.5725439389546711
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,2,128,1,float16,fp8,32767,0.19334399700164795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,7,0.008000000069538752
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,15,0.007983999947706858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,7,0.013199999928474426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,31,0.008069333309928576
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,31,0.013327999661366144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,63,0.008223999912540117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,15,0.01314666618903478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,131071,3.3316214879353843
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,127,0.008527999743819237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,63,0.013338666409254074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,255,0.010938666760921478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,127,0.013877333452304205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,3,0.012437333663304647
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1023,0.02516266703605652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,255,0.014576000471909841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,511,0.015482666591803232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,511,0.01545599972208341
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,2047,0.04378133515516917
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,1023,0.0182239996890227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,4095,0.19006399313608804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,2047,0.02334933231274287
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,4095,0.033344000577926636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,8191,0.19362133741378784
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,16383,0.3699893156687419
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,16383,0.7277866999308268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,8191,0.41344531377156574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,32767,0.7243946393330892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,32767,1.445247968037923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,65535,1.5472639401753743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,65535,3.275610605875651
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,8,8,128,1,float16,fp8,131071,3.0824639002482095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,131071,6.526741027832031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,1,0.03545066714286804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,1,0.030058667063713074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,3,0.036277333895365395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,3,0.03018666555484136
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,7,0.03667200108369192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,7,0.030207999050617218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,15,0.03669866671164831
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,15,0.030159999926884968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,31,0.036933332681655884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,31,0.030623999734719593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,63,0.03782399992148081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,63,0.03035199890534083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,127,0.04011200120051702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,127,0.03139200061559677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,255,0.19991999864578247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,255,0.04167466859022776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,511,0.3841013511021932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,1,128,1,float16,fp8,511,0.19873599211374918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,1,0.06728533407052358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,1,0.050399998823801674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,3,0.06836799780527751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,3,0.05016533533732096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,7,0.06922666728496552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,15,0.050554667909940086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,15,0.0690773328145345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,7,0.05067199965318044
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,31,0.07028266787528992
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,31,0.05054399867852529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,63,0.07157866656780243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,63,0.05106133222579956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,127,0.20552533864974976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,127,0.05380799869696299
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,255,0.39349865913391113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,255,0.20221867163976034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,1,0.11963733037312825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,511,0.7514932950337728
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,1,0.09141332904497783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,3,0.12196800112724304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,3,0.09162132938702901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,7,0.12384000420570374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,2,128,1,float16,fp8,511,0.38685333728790283
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,7,0.09130133191744487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,15,0.12404266993204753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,15,0.0915786623954773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,31,0.1253493328889211
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,31,0.09231999516487122
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,63,0.2142560084660848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,127,0.41015998522440594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,63,0.09270399808883667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,127,0.20622400442759195
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,255,0.7853333155314127
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,255,0.39070932070414227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,1,0.21449599663416544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,1,0.17005334297815958
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,3,0.218725323677063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,4,128,1,float16,fp8,511,0.7988426685333252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,511,1.6944160461425781
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,3,0.17044800519943237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,7,0.17019200325012207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,15,0.22128534317016602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,15,0.17062399784723917
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,63,0.3973013162612915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,7,0.22211732467015585
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,63,0.2078346610069275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,31,0.24731733401616415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,127,0.7878560225168864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,31,0.17206400632858276
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,127,0.3972906668980916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,255,1.6804320017496746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,255,0.7565279801686605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,1,0.05171733101209005
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,1,0.062234664956728615
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,511,3.3406718571980796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,3,0.06311466793219249
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,8,8,128,1,float16,fp8,511,1.5685920715332031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,3,0.05133866767088572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,15,0.06439466774463654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,7,0.05170666674772898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,7,0.06410666803518932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,31,0.051818668842315674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,63,0.05234666665395101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,127,0.2091360092163086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,15,0.05161599814891815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,63,0.06659199794133504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,127,0.05563200016816457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,255,0.39667733510335285
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,1,128,1,float16,fp8,255,0.20648000637690225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,31,0.06520533561706543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,1,0.12324266632397969
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,1,0.09105066458384196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,3,0.12507200241088867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,3,0.09126933415730794
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,7,0.12733866771062216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,7,0.09139200051625569
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,15,0.1269493301709493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,31,0.13198933005332947
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,31,0.09162132938702901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,63,0.21972266832987467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,63,0.09306666254997253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,127,0.4121866623560588
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,15,0.09187199672063191
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,127,0.21178666750590006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,255,0.7738080024719238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,1,0.22950933376948038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,1,0.1760693391164144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,2,128,1,float16,fp8,255,0.40065598487854004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,3,0.23410133520762125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,7,0.237936000029246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,7,0.1765600045522054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,3,0.17681600650151572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,15,0.17654933532079062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,31,0.27007466554641724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,15,0.2424373428026835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,63,0.4323999881744385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,31,0.1777120033899943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,127,0.8547946612040201
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,63,0.22441067298253378
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,127,0.41342933972676593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,1,0.42397868633270264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,1,0.3341226577758789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,255,1.7215040524800618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,4,128,1,float16,fp8,255,0.7746559778849283
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,3,0.432096004486084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,3,0.33418134848276776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,7,0.4424266815185547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,7,0.3349813222885132
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,15,0.4739146629969279
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,63,0.8018293380737305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,15,0.3358986775080363
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1,0.011168000598748526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,31,0.488591988881429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,31,0.34836800893147785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,3,0.008869333192706108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,63,0.44942935307820636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,7,0.008933333059151968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,15,0.008901333436369896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,31,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,31,0.011055999745925268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,63,0.009002666920423508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,63,0.011029332876205444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,3,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,127,0.009130666653315226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,127,0.011215999722480774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,255,0.011946666985750198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,255,0.012400000045696894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,7,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,511,0.013514666507641474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,511,0.012810666114091873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1023,0.014101333916187286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,1023,0.01657066618402799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,2047,0.022570667167504627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,2047,0.01995733380317688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,4095,0.02499199906984965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,4095,0.03446399917205175
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,8191,0.03463999927043915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,8191,0.05839466551939646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,16383,0.1949066718419393
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,16383,0.10601600011189778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,32767,0.37677868207295734
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,32767,0.21296000480651855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,1,128,1,float16,fp8,65535,0.41327468554178876
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,65535,0.7365492979685465
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1,0.012128000458081564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1,0.011157333850860596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,3,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,127,1.68613862991333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,7,0.011071999867757162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,3,0.012133333832025528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,7,0.012373333175977072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,15,0.011130666981140772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,15,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,31,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,31,0.011114666859308878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,63,0.011264000087976456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,127,0.012682666381200155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,127,0.011450666934251785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,255,0.015109332899252573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,255,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,511,0.013690666606028875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,511,0.015872000406185787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1023,0.02125866711139679
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,1023,0.016879999389251072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,4095,0.040362666050593056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,2047,0.02276800076166789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,2047,0.02643200010061264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,4095,0.034917332231998444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,8191,0.19499200582504272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,16383,0.19141866763432822
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,8191,0.058746665716171265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,16383,0.3750133514404297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,127,0.8441759745279948
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,32767,0.37149866422017414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,32767,0.7338240146636963
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1,0.011312000453472137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,2,128,1,float16,fp8,65535,0.7654186884562174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,65535,1.61518398920695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,3,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,7,0.013418667018413544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,7,0.01137599969903628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,15,0.012485332787036896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,15,0.011264000087976456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,31,0.012565333396196365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,31,0.011322667201360067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,63,0.012576000144084295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,63,0.011440000186363855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,127,0.013104000439246496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,127,0.011802667131026586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,255,0.012671999633312225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,255,0.015637333194414776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,511,0.020773333807786305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,511,0.014240000396966934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,1023,0.017498667041460674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1023,0.025850666066010792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,2047,0.035445332527160645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,4095,0.03583466758330663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,2047,0.023658665517965954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,4095,0.19988266626993814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,8191,0.38493335247039795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,8191,0.19140267372131348
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,16383,0.36737600962320965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,16383,0.7517973581949869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,32767,1.6688480377197266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,32767,0.7937013308207194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,63,0.012554666648308435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1,0.007983999947706858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1,0.013530666629473368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,3,0.008053333188096682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,7,0.008122666428486506
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,65535,3.3323678970336914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,3,0.01332266628742218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,7,0.013338666409254074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,4,128,1,float16,fp8,65535,1.574944019317627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,15,0.008127999802430471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,15,0.01328533391157786
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,31,0.013317332913478216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,31,0.008383999889095625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,63,0.01369599997997284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,63,0.008485333373149237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,3,0.012538666526476542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,127,0.014250667144854864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,127,0.00897066667675972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,255,0.011391999820868174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,255,0.015615999698638916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1023,0.026357332865397137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,511,0.01828266680240631
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,511,0.01628799984852473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,2047,0.19143466154734293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,2047,0.03398400048414866
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,1023,0.02330133318901062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,4095,0.37542398770650226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,8191,0.37114667892456055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,4095,0.19723200798034668
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,16383,0.7672426700592041
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,8191,0.7426186402638754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,16383,1.6619893709818523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1,0.00874133345981439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1,0.011066666493813196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,32767,1.545738697052002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,32767,3.326709429423014
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,3,0.008858666444818178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,3,0.011125333607196808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,7,0.008901333436369896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,15,0.008858666444818178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,15,0.011039999624093374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,7,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,31,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,63,0.008976000050703684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,63,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,31,0.009039999917149544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,8,8,128,1,float16,fp8,65535,3.0813493728637695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,127,0.009306666751702627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,127,0.011407999942700068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,255,0.012879999975363413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,255,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,511,0.01597333326935768
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1023,0.018629333625237148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,1023,0.021829334398110706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,511,0.013466666142145792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,2047,0.02369066576162974
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,2047,0.03382399926582972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,65535,6.642943700154622
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,4095,0.037765334049860634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,4095,0.05758399764696757
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,8191,0.10548266768455505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,8191,0.21301867564519247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,16383,0.373146653175354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,32767,0.4129600127538045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,32767,0.7302079995473226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,16383,0.22006932894388834
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1,0.011285333583752314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,3,0.01239466667175293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,3,0.01128000020980835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,7,0.012719999998807907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,7,0.011338666081428528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,15,0.012458667159080505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,8,8,128,1,float16,fp8,255,1.6014506022135417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,15,0.011328000575304031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,31,0.011274666835864386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,63,0.012608000387748083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,31,0.012549333274364471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,63,0.011445333560307821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,127,0.01173866664369901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,255,0.01573333392540614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,255,0.01328533391157786
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,511,0.020842666427294414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,511,0.0162773331006368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1023,0.025813333690166473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,1023,0.02235200007756551
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,2047,0.035589332381884255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,4095,0.19380267461140951
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,2047,0.03443199892838796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,1,128,1,float16,fp8,65535,0.8236160278320312
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,255,3.35698668162028
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,4095,0.05840533475081126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,8191,0.3736960093180339
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,8191,0.19164800643920898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,16383,0.8158026536305746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,16383,0.37028801441192627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,32767,1.4405333201090496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1,0.008277333031098047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1,0.011594666788975397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,3,0.008517333616813024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,3,0.011557333171367645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,7,0.008581333483258883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,7,0.011557333171367645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,15,0.008330666770537695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,65535,1.6246132850646973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,15,0.011653333902359009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,31,0.009583999713261923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,63,0.008432000254591307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,63,0.011781333635250727
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,31,0.011770666887362799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,127,0.009205333267649015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,127,0.011999999483426413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,255,0.011567999919255575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,255,0.013898666948080063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,511,0.016234666109085083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,511,0.016800000021855038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,1023,0.023168000082174938
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1023,0.026399999856948853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,2047,0.03565866748491923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,2047,0.1920213301976522
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,8191,0.36735999584198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,8191,0.7906453609466553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,4095,0.19100266695022583
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,4095,0.37508801619211835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,16383,0.783514658610026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,16383,1.6579519907633464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,65535,1.607968012491862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,65535,3.2237494786580405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,32767,1.5755573908487956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1,0.008559999987483025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,127,0.013151999562978745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,3,0.011594666788975397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,32767,3.321706771850586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,7,0.009519999846816063
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,7,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,3,0.008581333483258883
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,31,0.008799999952316284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,31,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,15,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,63,0.008672000219424566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,63,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,127,0.009114666531483332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,127,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,15,0.00874133345981439
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,255,0.011792000383138657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,255,0.012026666353146235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,511,0.011621333658695221
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,511,0.012522666404644648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1023,0.01301866645614306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,1023,0.011930666863918304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,2047,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,2047,0.013946666071812311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,4095,0.016480000068744022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,8191,0.02051199972629547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,4095,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,8191,0.013642666240533194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,16383,0.028245332340399425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,16383,0.01692266638080279
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,32767,0.03187733391920725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,4,128,1,float16,fp8,65535,3.1229546864827475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,65535,0.03685333331425985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,65535,0.033301333586374916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,131071,0.04757866760094961
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,131071,0.05585066477457682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1,0.010687999427318573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,3,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,7,0.008634666601816813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,4,2,128,1,float16,fp8,32767,0.7269173463185629
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,3,0.008527999743819237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,7,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,15,0.008752000207702318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,31,0.008693333094318708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,15,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,31,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,127,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,63,0.010933333386977514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,127,0.011018666128317514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,63,0.008816000074148178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,255,0.012234666695197424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,255,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,511,0.01258133351802826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,511,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1023,0.012954667210578918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,1023,0.01180800050497055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,2047,0.012128000458081564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,2047,0.013973332941532135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,4095,0.016501333564519882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,4095,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,8191,0.02109333376089732
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1,0.00843733362853527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,16383,0.016917333006858826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,32767,0.038917332887649536
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,16383,0.0332640012105306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,32767,0.023077333966890972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,8191,0.014042666802803675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,65535,0.03397866586844126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,65535,0.04866666595141093
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1,0.007829333345095316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,2,128,1,float16,fp8,131071,0.0558240016301473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,131071,0.203274667263031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1,0.01099733387430509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,3,0.007743999982873599
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,15,0.007776000226537387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,3,0.011029332876205444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,7,0.007663999994595845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,7,0.010842667271693548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,15,0.010858666151762009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,31,0.007877333089709282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,63,0.007877333089709282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,63,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,31,0.011114666859308878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,127,0.008031999692320824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,127,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,255,0.010277333358923594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,511,0.01522133375207583
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,511,0.011871999750534693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,255,0.011706666400035223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1023,0.024373332659403484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,2047,0.04293866455554962
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,1023,0.011839999506870905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,4095,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,4095,0.08062399923801422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,2047,0.012416000167528788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,8191,0.1548853317896525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,16383,0.3040320078531901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,8191,0.014245333770910898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,16383,0.017808000246683758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,32767,0.02420799930890401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,32767,0.6029920180638632
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,65535,0.03468266626199087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1,0.00854399986565113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1,0.010565333068370819
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,65535,1.2168052991231282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,65535,6.636821111043294
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,4,128,1,float16,fp8,131071,0.19155200322469076
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,3,0.010586666564146677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,7,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,3,0.008693333094318708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,15,0.010666667173306147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,31,0.008762666955590248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,31,0.010650667051474253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,4,1,128,1,float16,fp8,32767,0.022805333137512207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,15,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,131071,2.4357226689656577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,127,0.009008000294367472
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,63,0.00871999996403853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,63,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,127,0.010901333143313726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,255,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,511,0.012383999923865
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1023,0.01314666618903478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,1023,0.011525332927703857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,511,0.011525332927703857
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,255,0.01184533288081487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,2047,0.014165333161751429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,2047,0.011839999506870905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,4095,0.016762666404247284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,4095,0.013338666409254074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,8191,0.0207893339296182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,16383,0.034154665966828666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,7,0.010437333335479101
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,8191,0.016384000579516094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,16383,0.0223786657055219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,32767,0.03986666599909464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,32767,0.03465066601832708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,65535,0.048800001541773476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1,0.011871999750534693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,65535,0.05693866809209188
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1,0.010746666540702185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,1,128,1,float16,fp8,131071,0.10140800476074219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,3,0.01211200033624967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,131071,0.2259946664174398
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,7,0.012080000092585882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,3,0.010656000425418219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,7,0.010559999694426855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,15,0.012175999581813812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,15,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,31,0.01393066719174385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,63,0.012266666938861212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,63,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,31,0.010650667051474253
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,127,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,255,0.014767999450365702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,127,0.012400000045696894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,511,0.015013333410024643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,255,0.011653333902359009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,511,0.011653333902359009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,2047,0.01708799973130226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,1023,0.011786667009194693
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,2047,0.012037333101034164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,4095,0.01969066634774208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1023,0.01545599972208341
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,4095,0.01351999988158544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,8191,0.02852799991766612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,16383,0.033285332222779594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,8191,0.016762666404247284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,16383,0.022543999056021374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,32767,0.03468266626199087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,32767,0.04276266694068909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,65535,0.20000000794728598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,65535,0.05739733576774597
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,131071,0.38366933663686115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,2,128,1,float16,fp8,131071,0.1916853388150533
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,3,0.007690666864315669
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1,0.010911999891201654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,3,0.010735999792814255
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1,0.0075093333919843035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,7,0.007781333600481351
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,15,0.007743999982873599
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,7,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,63,0.007823999971151352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,15,0.010645333677530289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,31,0.010837333897749582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,63,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,31,0.007930666829148928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,127,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,127,0.008127999802430471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,511,0.015013333410024643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,255,0.01184533288081487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1023,0.02439466615517934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,255,0.010351999973257383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,2047,0.0430026650428772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,511,0.011978667229413986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,2047,0.012469333906968435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,1023,0.012133333832025528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,4095,0.08037866652011871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,8191,0.15493333339691162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,8191,0.01717866708834966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,4095,0.014021333307027817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,16383,0.30453334252039593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,16383,0.023365333676338196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,32767,0.6132853428522745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,65535,0.19113065799077353
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,32767,0.03638399889071783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,4,4,128,1,float16,fp8,131071,0.3680266539255778
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,65535,1.2189599672953289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1,0.008858666444818178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1,0.01126933346192042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,3,0.008992000172535578
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,7,0.00898133342464765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,131071,2.43340269724528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,3,0.01137599969903628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,15,0.009029333169261614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,7,0.011365332951148352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,15,0.01128000020980835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,31,0.009077333534757296
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,31,0.011296000331640244
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,127,0.009663999701539675
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,63,0.011514666179815928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,63,0.009226666763424873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,127,0.011802667131026586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,255,0.013269333789745966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,255,0.01332266628742218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,511,0.018165333817402523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,511,0.01637866720557213
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1023,0.023311999936898548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,1023,0.022218666970729828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,2047,0.034448000291983284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,2047,0.0330826664964358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,8191,0.19312532742818198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,8191,0.3739573160807292
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,4095,0.21316800514856973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,16383,0.37138664722442627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,4095,0.05835199852784475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1,0.012869333227475485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,16383,0.7306346893310547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1,0.011706666400035223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,3,0.011461333682139715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,7,0.011578666667143503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,15,0.01303999995191892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,15,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,3,0.012965332716703415
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,31,0.012986666212479273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,31,0.011472000430027643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,7,0.013034666577974955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,63,0.013370666652917862
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,63,0.011877333124478659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,1,128,1,float16,fp8,32767,0.7271413008371989
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,127,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,127,0.013797332843144735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,255,0.020762667059898376
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,255,0.013861333330472311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,511,0.017055999487638474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,511,0.02586666742960612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1023,0.035642666121323906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,4095,0.3737866481145223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,1023,0.02309333284695943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,4095,0.19260799884796143
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,32767,1.4435146649678547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,8191,0.7290933132171631
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,2047,0.03542399903138479
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,2047,0.19454934199651083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,8191,0.36793065071105957
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,16383,0.794373353322347
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,3,0.011722666521867117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1,0.013744000345468521
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1,0.011546666423479715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,16383,1.4406240781148274
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,3,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,7,0.011989332735538483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,32767,3.2155094146728516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,2,128,1,float16,fp8,32767,1.5657119750976562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,7,0.013637332866589228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,15,0.011909333368142446
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,15,0.013669333110253016
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,31,0.013642666240533194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,31,0.012229333321253458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,63,0.012378666549921036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,127,0.014917333920796713
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,127,0.0136266661187013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,255,0.017685333887736004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,255,0.018079999834299088
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,511,0.027888000011444092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,511,0.02293333411216736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,63,0.01444799949725469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,2047,0.19378666083017984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,2047,0.377834677696228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1023,0.19284266233444214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,1023,0.03291733314593633
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,8191,0.7267253398895264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,4095,0.745797316233317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1,0.010922666639089584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,4095,0.37096532185872394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1,0.008538666491707167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,3,0.008714666590094566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,8191,1.5764427185058594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,3,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,7,0.008725333337982496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,7,0.010890666395425797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,15,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,15,0.008698666468262672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,31,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,31,0.00871999996403853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,16383,3.3305174509684243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,63,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,127,0.009050666665037474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,127,0.010933333386977514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,63,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,255,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,255,0.011813333878914515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,511,0.012560000022252401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,511,0.011866666376590729
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1023,0.01322666679819425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,2047,0.013514666507641474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,2047,0.014698666830857595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,1023,0.01201066623131434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,4095,0.017290666699409485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,4095,0.01647466669480006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,8191,0.026176000634829204
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,8191,0.022554665803909302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,16383,0.031290667752424874
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,16383,0.034314667185147606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,32767,0.05806933343410492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,32767,0.04109866668780645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,65535,0.10590933760007222
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,65535,0.22046933571497598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,131071,0.3925439914067586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,1,128,1,float16,fp8,131071,0.21385600169499716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,16383,1.438704013824463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1,0.012149333953857422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,3,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,7,0.012319999436537424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,3,0.011007999380429586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,7,0.011157333850860596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,15,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,31,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,127,0.012618667135636011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,31,0.010933333386977514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,127,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,63,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,255,0.011839999506870905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,511,0.015311999867359797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,255,0.01492799942692121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,511,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,1023,0.012165332833925882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1023,0.015930666277805965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,4095,0.024405332903067272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,2047,0.01762666677435239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,2047,0.01461333284775416
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,15,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,4095,0.016693333784739178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,8191,0.02942399928967158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,16383,0.03477333237727483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,16383,0.03940266619126002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,8191,0.02269333352645238
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,32767,0.05852266649405161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,32767,0.21849066019058228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,65535,0.37905065218607586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,65535,0.19145600001017252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1,0.00766933336853981
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,2,128,1,float16,fp8,131071,0.37112534046173096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,3,0.007749333356817563
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,131071,0.8150826295216879
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,3,0.011183999478816986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,15,0.007802666475375493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,7,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,7,0.007781333600481351
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,31,0.00786666696270307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,15,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,31,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,127,0.00816000004609426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,63,0.011018666128317514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,63,0.007861333588759104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,4,4,128,1,float16,fp8,32767,3.087439854939779
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,255,0.010784000158309937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,511,0.015098666151364645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,255,0.012096000214417776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,127,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,511,0.012229333321253458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,2047,0.04314666489760081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,1023,0.012693333129088083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,2047,0.014261333892742792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,4095,0.017279999951521557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,4095,0.0804746647675832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,8191,0.15522133310635886
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,16383,0.3118613362312317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,8191,0.0235359991590182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,16383,0.03626666714747747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,32767,0.21207465728123984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,32767,0.6159253517786661
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,65535,0.368122657140096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,65535,1.2263466517130535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,4,4,128,1,float16,fp8,131071,0.7946986357371012
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,131071,2.449066638946533
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1,0.011834666132926941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1,0.01044800008336703
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,32767,5.901007970174153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,3,0.009306666751702627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,3,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1023,0.024485332270463307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,7,0.01190399999419848
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,15,0.009466666728258133
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,7,0.009514666472872099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,31,0.009482666850090027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,31,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,63,0.009722666814923286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,63,0.012085333466529846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,127,0.010485333700974783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,127,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,255,0.01834133391578992
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,255,0.013978666315476099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,511,0.02314666658639908
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,511,0.017370666066805523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,15,0.01180800050497055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1023,0.03316800047953924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,1023,0.023423999547958374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,2047,0.03591466695070267
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,2047,0.1933493415514628
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,8191,0.7302026748657227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,8191,0.40565868218739826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,4095,0.37445334593455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,4095,0.19311465819676718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,3,0.013823999712864557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,3,0.01691199963291486
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,1,128,1,float16,fp8,16383,0.7254933516184489
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1,0.014005333185195923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1,0.01655999943614006
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,15,0.016832000265518825
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,16383,1.4446667035420735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,15,0.013904000322024027
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,63,0.017322666943073273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,31,0.013967999567588171
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,7,0.016810666769742966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,31,0.01716800034046173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,63,0.01443733274936676
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,127,0.015237333873907724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,7,0.013722666849692663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,255,0.023381332556406658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,255,0.01801066721479098
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,127,0.018570666511853535
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,511,0.03306133300065994
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,511,0.023168000082174938
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1023,0.19264533122380575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,1023,0.03342933456103007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,4095,0.37089065710703534
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,4095,0.7267093658447266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,2047,0.19413334131240845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1,0.019760000209013622
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1,0.019098666807015736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,8191,0.7273706595102946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,2047,0.4113493363062541
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,8191,1.4409759839375813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,3,0.019797333826621372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,7,0.019589333484570186
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,7,0.019776000330845516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,15,0.019813333948453266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,31,0.01972266659140587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,31,0.019871999820073444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,16383,2.8557119369506836
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,3,0.0191040001809597
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,15,0.019354666272799175
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,63,0.020031999796628952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,2,128,1,float16,fp8,16383,1.430789311726888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,63,0.020400000115235645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,127,0.02165866643190384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,127,0.021802666286627453
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,255,0.031130666534105938
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,255,0.026693334182103474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,511,0.19357333580652872
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,2047,0.7476480007171631
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,2047,0.3734399875005086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1023,0.3797173500061035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,4095,1.484373410542806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,511,0.03667200108369192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,8191,1.5563252766927083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,4095,0.7860799630482992
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,1023,0.19324799378712973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1,0.013253333667914072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1,0.01422400027513504
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,3,0.013280000537633896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,3,0.014074667046467463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,7,0.014202666779359182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,7,0.013327999661366144
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,8191,2.9615306854248047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,15,0.013418667018413544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,15,0.014101333916187286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,31,0.013546666751305262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,31,0.014389333625634512
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,63,0.013829333086808523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,127,0.015098666151364645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,127,0.015418666104475657
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,4,4,128,1,float16,fp8,16383,2.8613173166910806
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,16383,6.65663464864095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,63,0.014746667196353277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,255,0.019685332973798115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,255,0.01850133389234543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,511,0.029418667157491047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,511,0.02346133440732956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1023,0.19000534216562906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1,0.02399466683467229
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1,0.020181333025296528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,2047,0.19537067413330078
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,3,0.020288000504175823
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,3,0.024314666787783306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,7,0.024832000335057575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,1023,0.03489600121974945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,15,0.024586667617162068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,15,0.020341333001852036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,2047,0.41120533148447674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,7,0.020288000504175823
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,31,0.02514133354028066
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,1,128,1,float16,fp8,4095,0.3736319939295451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,4095,0.7867466608683268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,63,0.025370667378107708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,31,0.020330666253964107
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,63,0.020581333587567013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,127,0.022042666872342426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,255,0.03719999889532725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,511,0.03676266719897588
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,127,0.027077332139015198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,1023,0.19806400934855142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1023,0.3731093406677246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,511,0.1932906707127889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,4095,1.4354772567749023
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,255,0.027232001225153606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,4095,0.7317972977956136
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,3,0.03193599979082743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,2,128,1,float16,fp8,2047,0.3772746721903483
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,2047,0.7698293526967367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,7,0.03414933383464813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,3,0.03342399994532267
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1,0.0329066663980484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,7,0.031850665807724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1,0.03193599979082743
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,15,0.03427733232577642
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,31,0.03215466688076655
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,63,0.03510933369398117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,63,0.032111999889214836
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,127,0.037503999968369804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,127,0.03279466678698858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,255,0.1946666638056437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,255,0.043354665239652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,31,0.034714666505654655
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,15,0.03207999964555105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,2047,1.4948372840881348
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,511,0.19570134083429971
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,511,0.38206934928894043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,2047,0.7304213047027588
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,1023,0.37786134084065753
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1,0.02060266708334287
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1023,0.8214933077494303
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,4,4,128,1,float16,fp8,4095,1.5554614067077637
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,4095,3.3135201136271157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1,0.020784000555674236
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,3,0.020842666427294414
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,3,0.02109866589307785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,7,0.021018666525681812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,31,0.021370666722456615
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,31,0.020997333029905956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,15,0.020917333662509918
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,63,0.022117334107557934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,63,0.021317332983016968
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,15,0.021333334346612293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,7,0.021173333128293354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,127,0.023546665906906128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,127,0.022458667556444805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,255,0.03363200028737386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,255,0.027994667490323383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,511,0.03779733429352442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,511,0.19158399105072021
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1023,0.3720373312632243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,1023,0.1970400015513102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1,0.03890133400758108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1,0.03270400067170461
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,1,128,1,float16,fp8,2047,0.3772159814834595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,2047,0.7716480096181234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,3,0.03951466580231985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,3,0.032645332316557564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,15,0.04008000095685323
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,7,0.03997333347797394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,15,0.032602667808532715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,7,0.032629333436489105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,63,0.041264000038305916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,63,0.03277866790692011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,127,0.04357333481311798
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,127,0.03347733368476232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,255,0.20670932531356812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,255,0.04446400205294291
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,31,0.040250666439533234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,31,0.03271466741959254
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,511,0.37912531693776447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1,0.05896000067392985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1,0.05336533486843109
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,511,0.19805334011713663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,3,0.053530668218930565
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1023,0.7762133280436198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,1023,0.3816639979680379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,7,0.06101333101590475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,3,0.060266668597857155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,7,0.053674668073654175
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,2,128,1,float16,fp8,2047,0.7362346649169922
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,15,0.06081599990526835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,2047,1.612826665242513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,31,0.053642665346463524
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,63,0.06322666505972545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,63,0.05442133545875549
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,127,0.19730132818222046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,31,0.06206933160622915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,15,0.053557331363360085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,255,0.19992534319559732
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,511,0.7550240357716879
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,2047,3.183919906616211
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,127,0.056320001681645714
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,511,0.40535998344421387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,255,0.3850719928741455
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1,0.011077333241701126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,7,0.010784000158309937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,15,0.008767999708652496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,15,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,31,0.009029333169261614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,3,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,31,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,63,0.008986666798591614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,63,0.010911999891201654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,127,0.009392000113924345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,127,0.010954666882753372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,255,0.012191999703645706
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,2047,1.4488213857014973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,255,0.011770666887362799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,511,0.012618667135636011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1023,0.013434667140245438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,1023,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,7,0.008943999807039896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,2047,0.015194666882356008
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,2047,0.016415999581416447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,4095,0.022367998957633972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,4095,0.021909333765506744
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,8191,0.026943999032179516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1,0.008762666955590248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,511,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,16383,0.05834133426348368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,8191,0.034389334420363106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,16383,0.0366293340921402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,32767,0.19523199399312338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,32767,0.10585600137710571
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,65535,0.21389333407084146
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,65535,0.37886401017506915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,131071,0.41464531421661377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1,0.012133333832025528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1,0.010741333166758219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,3,0.012298667182525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,131071,0.780623992284139
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,3,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,1,128,1,float16,fp8,3,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,7,0.012256000190973282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,7,0.010677333921194077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,15,0.012304000556468964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,15,0.010837333897749582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,31,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,63,0.012389333297808966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,31,0.012416000167528788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,127,0.012639999389648438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,255,0.014853333433469137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,127,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,255,0.011834666132926941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,511,0.015439999600251516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,511,0.012080000092585882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1023,0.016469333320856094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,1023,0.013466666142145792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,2047,0.022202665607134502
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,2047,0.01674666628241539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,4095,0.027290667096773785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,4095,0.022597332795461018
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,8191,0.03678400069475174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,8191,0.0347626656293869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,16383,0.0584746648867925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,16383,0.19504000743230185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,32767,0.3745439847310384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,32767,0.19180800517400107
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,65535,0.37169599533081055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1,0.007743999982873599
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,65535,0.8091200192769369
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,3,0.007813333223263422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,3,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,7,0.007941333577036858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,2,128,1,float16,fp8,131071,0.784005324045817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,7,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,15,0.007989333321650824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,131071,1.6198293368021648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1023,1.6667466163635254
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,63,0.00790933333337307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,31,0.011120000233252844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,31,0.007920000081261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,127,0.008373333141207695
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,63,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,255,0.010645333677530289
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,127,0.011498666057984034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,4,4,128,1,float16,fp8,1023,0.788703997929891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,511,0.012495999534924826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,511,0.015253332753976187
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,2047,0.04329599936803182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1023,0.024666666984558105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,1023,0.013989333063364029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,4095,0.08066666622956593
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,4095,0.023391999304294586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,8191,0.18661334117253622
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,8191,0.03584533433119456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,16383,0.19137599070866904
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,16383,0.36187732219696045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,32767,0.3667253255844116
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,32767,0.7946399847666422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,65535,0.7221600214640299
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,65535,1.6105599403381348
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,131071,1.5735680262247722
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,131071,3.1834932963053384
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,255,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,4,4,128,1,float16,fp8,2047,0.01721599946419398
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1,0.03509333233038584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1,0.02975466599067052
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,3,0.035786665976047516
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,3,0.029898665845394135
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,7,0.03623466690381368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,7,0.03002133220434189
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,15,0.03605333218971888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,15,0.029711998999118805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,31,0.03678400069475174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,31,0.029893333713213604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,63,0.03748266647259394
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,63,0.03002133220434189
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,127,0.03941333293914795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,127,0.031023999055226643
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,255,0.194815993309021
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,255,0.04117333392302195
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,511,0.37785065174102783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,511,0.19622933864593506
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1,0.06573866804440816
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1,0.04994133114814758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1023,0.77619735399882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,1,128,1,float16,fp8,1023,0.3782506783803304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,3,0.06666133304437001
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,3,0.0498986691236496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,7,0.06778666873772939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,7,0.05011733373006185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,15,0.06762666503588359
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,15,0.050213331977526345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,63,0.06993600229422252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,63,0.05067733426888784
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,31,0.06901866694291432
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,127,0.1985386610031128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,31,0.05050666630268097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,127,0.05318933228651682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,255,0.3835253318150838
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,255,0.19893866777420044
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1,0.11226666967074077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,511,0.7393439610799154
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,511,0.38224534193674725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1,0.09083200494448344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,3,0.09109866619110107
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1023,1.6130879720052083
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,3,0.11401066184043884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,2,128,1,float16,fp8,1023,0.7892266909281412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,7,0.1160640021165212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,15,0.09113599856694539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,7,0.09078400333722432
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,15,0.11547733346621196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,31,0.11759466926256816
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,63,0.20138132572174072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,127,0.39100801944732666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,127,0.20494399468104044
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,255,0.7796479860941569
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,255,0.38581868012746173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,63,0.09293333689371745
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,31,0.09190932909647624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,511,1.6746506690979004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,511,0.7959360281626383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,1,0.06145066519578298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,3,0.062421331803003945
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,1,0.05037866532802582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,3,0.05070933202902476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,4,4,128,1,float16,fp8,1023,1.5581386884053547
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1023,3.3313652674357095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,15,0.0634080022573471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,7,0.06374933322270711
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,7,0.05064533154169718
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,31,0.0642080008983612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,15,0.050885334610939026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,31,0.05115733544031779
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,127,0.2010293404261271
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,127,0.053599998354911804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,63,0.05162133276462555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,63,0.06549333532651265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,255,0.19986132780710855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,255,0.4227199951807658
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,1,128,1,float16,fp8,511,0.38565866152445477
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,511,0.7980053424835205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,1,0.1202826698621114
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,1,0.09037333726882935
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,3,0.12204800049463908
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,3,0.09033067027727763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,7,0.12371200323104858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,7,0.09058133761088054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,15,0.1240053375562032
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,31,0.12688533465067545
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,31,0.09096533060073853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,63,0.20652800798416138
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,63,0.09184533357620239
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,15,0.09100799759229024
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,127,0.3954720099767049
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,127,0.2076586683591207
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,255,0.7775413195292155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,255,0.3919839859008789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,1,0.2183199922243754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,1,0.175546665986379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,511,1.4638932545979817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,2,128,1,float16,fp8,511,0.7477173010508219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,3,0.22251200675964355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,3,0.17548799514770508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,7,0.22665067513783774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,15,0.22684800624847412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,15,0.17622933785120645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,63,0.23298666874567667
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,31,0.2521386742591858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,7,0.1752906640370687
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,63,0.40090131759643555
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,127,0.7750399907430013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1,0.008799999952316284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,3,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,3,0.010922666639089584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,7,0.008805333326260248
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,15,0.008879999940594038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,15,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,31,0.008874666566650072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,31,0.01097600037852923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,63,0.00895999992887179
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,63,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,127,0.009178666397929192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,31,0.17677332957585654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,127,0.011109333485364914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,255,0.012309333930412928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,255,0.011855999628702799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,511,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1023,0.014138666292031607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,1023,0.01661866654952367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,2047,0.0199946661790212
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,511,0.013056000073750814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,127,0.4038399855295817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,2047,0.02233600119749705
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,4095,0.02492800106604894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,4095,0.03443199892838796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,8191,0.05834666887919108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,16383,0.1938986579577128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,16383,0.10636267066001892
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,32767,0.37389334042867023
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,32767,0.21262399355570474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,65535,0.7326080004374186
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,65535,0.41468799114227295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,131071,1.626815954844157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,1,128,1,float16,fp8,131071,0.8278559843699137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1,0.012335999558369318
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,8191,0.034832000732421875
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,3,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,3,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,7,0.01239466667175293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,7,0.010965333630641302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,15,0.012309333930412928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,31,0.012416000167528788
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,15,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,31,0.010928000013033548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,63,0.012495999534924826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,63,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,127,0.012560000022252401
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,255,1.6856746673583984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,127,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,255,0.01515199989080429
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,255,0.012074666718641916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,511,0.01598400001724561
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,511,0.0136266661187013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,1023,0.01674666628241539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,2047,0.022698665658632915
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,2047,0.026341333985328674
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,4095,0.04021333406368891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,4095,0.03473600000143051
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,16383,0.19223999977111816
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,16383,0.3731146653493245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,8191,0.19428267081578574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,8191,0.05870933334032694
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,32767,0.7313546339670817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,32767,0.3782293399175008
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,65535,0.8029066721598307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,65535,1.6155519485473633
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1,0.007813333223263422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,3,0.00789866658548514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,3,0.011221333096424738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,2,128,1,float16,fp8,131071,1.606895923614502
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,7,0.008053333188096682
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,7,0.011114666859308878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,131071,3.2240959803263345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,15,0.011317333827416102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,31,0.008063999935984612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,31,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,15,0.008416000132759413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,63,0.008197333042820295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,63,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,127,0.008432000254591307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,255,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,127,0.011781333635250727
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,255,0.01259200026591619
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,511,0.014127999544143677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,511,0.015562667200962702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1023,0.021365332106749218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,1023,0.01718933383623759
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1023,0.02499199906984965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,2047,0.04393066465854645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,2047,0.023573334018389385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,4095,0.18871466318766275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,4095,0.035904000202814736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,8191,0.19138665994008383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,8191,0.3655466636021932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,16383,0.7244906425476074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,16383,0.3677866856257121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,32767,0.793562650680542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,32767,1.6288053194681804
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1,0.00873066671192646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,65535,1.574010690053304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,65535,3.2417065302530923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1,0.01097600037852923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,3,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,3,0.011045332998037338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,7,0.008896000062425932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,7,0.01102399950226148
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,15,0.008901333436369896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,31,0.008943999807039896
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,4,4,128,1,float16,fp8,131071,3.1240533192952475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,31,0.010970667004585266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,63,0.00897066667675972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,63,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,255,0.8152213096618652
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,127,0.011264000087976456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,255,0.01268799975514412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,131071,6.486202875773112
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,255,0.012800000607967377
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,511,0.01339200014869372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1023,0.0185759998857975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,127,0.009248000259200731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,2047,0.023754666248957317
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,2047,0.03442133218050003
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,4095,0.03754666695992152
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,4095,0.05760000149408976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,1023,0.02170666555563609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,8191,0.19218667348225912
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,8191,0.10527466734250386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,16383,0.2188160022099813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,16383,0.3722720146179199
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,32767,0.7283946673075358
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,32767,0.4139946699142456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,65535,1.6222933133443196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,511,3.3426507314046225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,4,4,128,1,float16,fp8,511,1.579429308573405
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,511,0.01609066625436147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1,0.008394666636983553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1,0.011253333340088526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,7,0.008266666904091835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,7,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,15,0.008186666915814081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,15,0.011477333803971609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,31,0.008117333054542542
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,31,0.011274666835864386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,127,0.008661333471536636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,63,0.011370666325092316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,63,0.008538666491707167
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,127,0.01180800050497055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,255,0.011007999380429586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,255,0.013232000172138214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,511,0.015770666301250458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,511,0.01621866722901662
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1023,0.025306666890780132
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,1023,0.022448000808556873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,2047,0.04404800136884054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,2047,0.034416000048319496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,4095,0.18518400192260742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,8191,0.1914773384730021
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,4095,0.05820266902446747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,8191,0.3600533405939738
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,65535,0.8273066679636637
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,16383,0.37721065680185956
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,16383,0.7611253261566162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,32767,0.8131679693857828
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,32767,1.5834399859110515
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,1,128,1,float16,fp8,131071,1.660853385925293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1,0.01081066702802976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,3,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,131071,3.236501375834147
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,7,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,15,0.008693333094318708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,15,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,7,0.008810666700204214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,31,0.008816000074148178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,31,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,63,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,63,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,127,0.009050666665037474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,127,0.010954666882753372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,3,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,255,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,255,0.011946666985750198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,511,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,511,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1023,0.01302933320403099
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,1023,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,3,0.008367999767263731
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,2047,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,2047,0.014127999544143677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,4095,0.016271999726692837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,4095,0.01210133358836174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,8191,0.013861333330472311
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,8191,0.020410666863123577
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,16383,0.02934933453798294
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,16383,0.016613333175579708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,65535,0.037061333656311035
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,32767,0.03179733455181122
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,32767,0.02351466566324234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,65535,0.03320533285538355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,131071,0.04736533264319102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1,0.010832000523805618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,1,128,1,float16,fp8,131071,0.05583466589450836
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,3,0.007834666719039282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,7,0.010981333752473196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,15,0.007925333455204964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1,0.007765333478649457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,15,0.010901333143313726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,31,0.008389333263039589
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,7,0.007903999959429106
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,31,0.010832000523805618
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,63,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,63,0.007962666451931
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,127,0.008112000301480293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,255,0.010288000106811523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,127,0.011066666493813196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,255,0.011887999872366587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,511,0.015103999525308609
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,511,0.01166933278242747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1023,0.024383999407291412
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,1023,0.011765333513418833
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,2047,0.04308266441027323
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,4095,0.08043733239173889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,2047,0.01198400060335795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,4095,0.012479999413092932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,8191,0.15516799688339233
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,8191,0.01393066719174385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,16383,0.30457599957784015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,16383,0.016976000120242436
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,32767,0.023317334552605946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,32767,0.6034026543299357
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,65535,0.033728001018365227
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,65535,1.2012746334075928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,2,2,128,1,float16,fp8,131071,0.056176001826922096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,131071,2.43176539738973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,3,0.00871999996403853
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,3,0.010853332777818045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,7,0.010858666151762009
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,7,0.00877333308259646
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,15,0.008853333070874214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,31,0.008709333216150602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,15,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,31,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,65535,1.6064213116963704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,63,0.011066666493813196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,127,0.00914666677514712
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,63,0.00884799969693025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,255,0.012240000069141388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,255,0.011658667276302973
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,511,0.01180800050497055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,511,0.01249066616098086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,127,0.010922666639089584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1023,0.012970666090647379
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,2047,0.014202666779359182
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,1023,0.01180800050497055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,2047,0.012144000579913458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,4095,0.01358933374285698
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,4095,0.01635733370979627
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,8191,0.021530665457248688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,8191,0.016586666305859882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,65535,3.1898880004882812
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,16383,0.02275199939807256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,16383,0.033189333975315094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,32767,0.04247466723124186
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,32767,0.03473066786924998
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,65535,0.04785066843032837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,131071,0.2264853318532308
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,131071,0.10192533334096272
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1,0.007776000226537387
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,3,0.010778666784365972
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,15,0.008565333361426989
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,3,0.007631999750932057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,7,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,1,128,1,float16,fp8,65535,0.05719466507434845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,15,0.01081066702802976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,63,0.007802666475375493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,31,0.010837333897749582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,63,0.011141333729028702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,31,0.008186666915814081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,255,0.010341333225369453
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,127,0.010970667004585266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,255,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,511,0.01166933278242747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,511,0.015114666273196539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1023,0.024282666544119518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,127,0.00821333316465219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,1023,0.01179733375708262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,2047,0.04301866888999939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,4095,0.08060266574223836
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,2047,0.01219733307758967
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,4095,0.01368533323208491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,8191,0.01676799977819125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,8191,0.15500799814860025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,16383,0.022810667753219604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,16383,0.3046346704165141
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,32767,0.03498133271932602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,32767,0.6033386786778768
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,65535,0.05755733450253805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,2,2,128,1,float16,fp8,131071,0.1925013264020284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,65535,1.2221439679463704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1,0.008986666798591614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1,0.011429333438475927
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,2,2,128,1,float16,fp8,131071,3.204751968383789
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,3,0.008885333314538002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,131071,2.4318453470865884
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,3,0.011258666714032492
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,7,0.008986666798591614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,7,0.01128000020980835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,15,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,31,0.00926399976015091
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,31,0.011354666203260422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,63,0.009205333267649015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,63,0.011338666081428528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,131071,6.340389251708984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,127,0.009546666716535887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,127,0.011802667131026586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,255,0.01331199953953425
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,255,0.013232000172138214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,511,0.018016000588734944
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,511,0.016362667083740234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,15,0.011461333682139715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,1023,0.022474666436513264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,2047,0.03278400003910065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,2047,0.034373333056767784
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,8191,0.19299733638763428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,8191,0.4142826795578003
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,4095,0.19239999850591025
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,4095,0.0580213318268458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1023,0.023306667804718018
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,16383,0.37065601348876953
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,16383,0.7299466927846273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1,0.007920000081261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1,0.011642667154471079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,32767,1.6194613774617512
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,32767,0.8028639952341715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,3,0.008336000144481659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,7,0.008938666433095932
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,3,0.011535999675591787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,15,0.011535999675591787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,7,0.01163200040658315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,15,0.008197333042820295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,31,0.012261333564917246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,31,0.011642667154471079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,63,0.00860799973209699
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,63,0.011887999872366587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,127,0.01239466667175293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,255,0.011717333147923151
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,127,0.009226666763424873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,255,0.01393066719174385
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,511,0.01640533283352852
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,511,0.01708799973130226
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1023,0.02644266684850057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,1023,0.023168000082174938
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,4095,0.3630239963531494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,4095,0.19103467464447021
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,2047,0.03557866563399633
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,2047,0.20563199122746786
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,8191,0.7176693280537924
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,8191,0.36796267827351886
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,16383,1.6065866152445476
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1,0.008656000097592672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,16383,0.7217386563618978
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,3,0.008746666833758354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,7,0.00890666681031386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,3,0.010751999914646149
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,7,0.010944000134865442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,15,0.011034666250149408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,15,0.008912000184257826
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,32767,3.20794677734375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,31,0.008858666444818178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,1,128,1,float16,fp8,65535,1.4415520032246907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,31,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,63,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,65535,3.062389373779297
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,127,0.010981333752473196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,255,0.01228800043463707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,511,0.012378666549921036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,511,0.011861333002646765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1023,0.013072000195582708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,1023,0.012015999605258306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,2047,0.014378666877746582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,65535,3.1134398778279624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,63,0.008874666566650072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,2047,0.013424000392357508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,4095,0.01699200024207433
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,8191,0.02239999920129776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,8191,0.02555199960867564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,16383,0.03054400036732356
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,127,0.009125333279371262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,16383,0.03439466655254364
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,32767,0.0405173326532046
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,2,2,128,1,float16,fp8,32767,1.433861255645752
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,32767,0.05820799867312113
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,65535,0.10591466228167216
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,255,0.011770666887362799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,65535,0.21964265902837118
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1,0.007727999861041705
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,131071,0.38229866822560626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,3,0.007680000116427739
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,3,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,7,0.007696000238259633
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,15,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,7,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,15,0.00772266648709774
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,65535,6.39079475402832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,31,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,63,0.007840000092983246
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,63,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,127,0.008112000301480293
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,31,0.007797333101431529
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,255,0.011727999895811081
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,255,0.010368000095089277
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,511,0.011920000116030375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,511,0.015040000279744467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1023,0.02425066630045573
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,4095,0.016352000335852306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,1023,0.012058666596810022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,2047,0.04302933315436045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,4095,0.08030933141708374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,2047,0.01357866699496905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,1,128,1,float16,fp8,131071,0.22035199403762817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,8191,0.15496533115704855
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,8191,0.022624000906944275
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,16383,0.034917332231998444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,16383,0.30451732873916626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,4095,0.016602666427691776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,32767,0.058373332023620605
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,32767,0.6138293345769247
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,65535,0.19142399231592813
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,65535,1.220965305964152
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,131071,0.41462934017181396
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1,0.008965333302815756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,131071,2.4340160687764487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1,0.011616000284751257
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,2,2,128,1,float16,fp8,127,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,3,0.009205333267649015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,3,0.011637333780527115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,7,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,7,0.009269333134094873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,15,0.009290666629870733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,31,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,15,0.011642667154471079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,31,0.00933333362142245
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,63,0.009712000067035357
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,63,0.01166933278242747
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,127,0.010106666634480158
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,127,0.012357333054145178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,255,0.018160000443458557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,255,0.014010666559139887
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,511,0.023168000082174938
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,511,0.016949333250522614
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1023,0.03253866732120514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,1023,0.023082666099071503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,2047,0.21248533328374228
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,2047,0.035802667339642845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,8191,0.728762706120809
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,8191,0.3699520031611125
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,4095,0.19202667474746704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1,0.011557333171367645
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,4095,0.41468799114227295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,16383,1.4446934064229329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1,0.013781332721312841
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,16383,0.725226640701294
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,3,0.013471999516089758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,15,0.011999999483426413
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,7,0.01360000049074491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,3,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,31,0.012058666596810022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,1,128,1,float16,fp8,32767,1.57151460647583
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,31,0.013653332988421122
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,7,0.012026666353146235
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,63,0.012341332932313284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,15,0.013546666751305262
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,63,0.014175999909639359
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,127,0.013440000514189402
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,32767,2.878351847330729
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,255,0.01834133391578992
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,255,0.017871999492247898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,511,0.02794133375088374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,511,0.02295999974012375
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,127,0.014991999914248785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1023,0.18592000007629395
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,4095,0.3697813351949056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,4095,0.7194613615671793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,1023,0.03336533407370249
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,8191,1.4315199851989746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,8191,0.7234400113423666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,2047,0.19340799252192178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,16383,1.5458827018737793
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1,0.012906666845083237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1,0.013850666582584381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,2047,0.40571733315785724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,16383,2.851754824320475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,3,0.013232000172138214
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,7,0.013237333546082178
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,7,0.013957332819700241
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,3,0.01379199946920077
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,15,0.013642666240533194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,31,0.013434667140245438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,15,0.01320533330241839
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,31,0.013834666460752487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,63,0.013909333695967993
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,63,0.014282666146755219
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,32767,6.402575810750325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,2,2,128,1,float16,fp8,32767,2.845151901245117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,127,0.014698666830857595
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,127,0.01525866612792015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,255,0.01972266659140587
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,255,0.01800000046690305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,511,0.029343999922275543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,511,0.022970666488011677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,2047,0.3686346610387166
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,2047,0.19553067286809286
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,8191,0.7283519903818766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,8191,1.443306605021159
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1023,0.209279994169871
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1,0.0198186660806338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,1023,0.03392533212900162
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1,0.019071999937295914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,3,0.019173332800467808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,4095,0.7248160044352213
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,1,128,1,float16,fp8,4095,0.37169599533081055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,7,0.01956266661485036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,3,0.019978666057189304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,7,0.019744000087181728
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,15,0.019962667177120846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,15,0.01945066700379054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,63,0.020175999651352566
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,63,0.020261333634455998
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,127,0.021776000658671062
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,127,0.02165333429972331
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,255,0.03186133255561193
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,255,0.026698666314284008
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,511,0.03622400015592575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,31,0.019765333582957584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,511,0.205295999844869
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,31,0.020026666422684986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,4095,0.7293972969055176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,1023,0.1952106753985087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1023,0.36500267187754315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,4095,1.4261333147684734
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,2047,0.804527997970581
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1,0.020501332978407543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,2047,0.4012426535288493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,2,2,128,1,float16,fp8,8191,1.5588639577229817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,8191,2.8407414754231772
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,3,0.020303999384244282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1,0.020389333367347717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,3,0.020703999946514767
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,7,0.020442667106787365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,31,0.020394666741291683
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,63,0.02179199953873952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,63,0.02080533280968666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,15,0.020421333611011505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,127,0.02349333216746648
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,31,0.021514666577180225
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,7,0.02107733239730199
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,15,0.021045332153638203
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,127,0.02195200075705846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,255,0.033600000043710075
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,255,0.027119999130566914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,511,0.2099519968032837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,511,0.037050666908423104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1023,0.3702346483866374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,1023,0.1954080065091451
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1,0.03321066747109095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1,0.03201066702604294
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,2047,0.37511467933654785
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,2047,0.8136959870656332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,7,0.03438399980465571
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,7,0.03219199925661087
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,3,0.03387733300526937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,3,0.03205333401759466
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,1,128,1,float16,fp8,4095,0.7318720022837321
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,31,0.03481066723664602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,15,0.03432533393303553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,4095,1.613968054453532
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,63,0.03585600107908249
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,31,0.03234133372704188
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,63,0.03232000023126602
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,15,0.032144000132878624
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,127,0.03297599901755651
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,255,0.1874986688296
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,511,0.3689173460006714
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,511,0.19746132691701254
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,1023,0.3776053190231323
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1023,0.8054827054341634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,255,0.04435733457406362
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,4095,1.4383254051208496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,127,0.037615999579429626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,3,0.011066666493813196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,7,0.010922666639089584
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,15,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,7,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,15,0.01097600037852923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,31,0.008832000195980072
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,3,0.008842666943868002
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,31,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,63,0.009109333157539368
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,63,0.011087999989589056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,127,0.009072000160813332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,127,0.01097600037852923
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,255,0.01227733368674914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,255,0.011855999628702799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,511,0.012618667135636011
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,511,0.01192533348997434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1023,0.013471999516089758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1023,0.013343999783198038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,4095,2.857167879740397
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,2047,0.01516266663869222
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,1,0.011029332876205444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1,0.008799999952316284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,2047,0.01648533344268799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,4095,0.02186133215824763
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,8191,0.02679466704527537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,8191,0.03436266630887985
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,16383,0.03659733384847641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,16383,0.05816533168156942
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,32767,0.19402132431666055
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,32767,0.10587732990582784
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,65535,0.21291200319925943
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,65535,0.3774506648381551
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1,0.007637333124876022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,131071,0.4168959856033325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1,0.011002667248249054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,131071,0.7583200136820475
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,3,0.007760000104705493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,1,128,1,float16,fp8,4095,0.02248000105222066
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,3,0.011178666104873022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,7,0.01101333275437355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,15,0.007791999727487564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,7,0.007818666597207388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,15,0.01099733387430509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,63,0.007920000081261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,31,0.010928000013033548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,63,0.010981333752473196
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,31,0.007914666707317034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,127,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,255,0.010442666709423065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,255,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,127,0.008218666538596153
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,511,0.015119999647140503
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1023,0.024458666642506916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,511,0.011946666985750198
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,1023,0.013429333766301474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,2047,0.016538667182127636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,4095,0.0804746647675832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,4095,0.02258133391539256
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,2047,0.043098668257395424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,8191,0.15520000457763672
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,8191,0.034703999757766724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,16383,0.0584853341182073
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,16383,0.31150933106740314
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,32767,0.19176000356674194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,32767,0.6171840031941732
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,65535,0.3703146775563558
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,65535,1.2252906958262126
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,2,2,128,1,float16,fp8,2047,0.731935977935791
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,2,2,128,1,float16,fp8,131071,0.8137919902801514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,2047,1.60480531056722
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,131071,2.445354620615641
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1,0.02958933264017105
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1,0.03525333354870478
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,3,0.03571200122435888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,7,0.03621866554021835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,15,0.03620799879233042
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,15,0.029845332105954487
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,3,0.02959466725587845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,7,0.0295413335164388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,31,0.03660800059636434
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,31,0.029829333225886028
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,63,0.037445334096749626
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,63,0.02991466720898946
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,127,0.03957333415746689
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,127,0.030794667700926464
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,255,0.19242666165033975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,255,0.04178133110205332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,511,0.37458133697509766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,511,0.1951626737912496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1023,0.8166986306508383
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1,0.059338668982187905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,1023,0.37538135051727295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1,0.0496373325586319
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,2047,1.6181920369466145
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,3,0.06043200194835663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,1,128,1,float16,fp8,2047,0.7839146455128988
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,3,0.050000001986821495
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,7,0.04960533479849497
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,15,0.0497920016447703
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,7,0.06141866743564606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,15,0.061109334230422974
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,31,0.050016000866889954
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,63,0.05087999999523163
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,127,0.19006933768590292
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,127,0.05212266743183136
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,255,0.3710293372472127
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,31,0.06262933214505513
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,255,0.19769599040349325
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,63,0.06365866462389629
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,511,0.8071946303049723
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,511,0.3779840071996053
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1023,1.4335360527038574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,1023,0.7333013216654459
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1,0.05031466484069824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1,0.06108800073464712
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,3,0.06228266656398773
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,2047,3.196048100789388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,3,0.05044800043106079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,2,2,128,1,float16,fp8,2047,1.5491092999776204
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,15,0.06316266457239787
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,15,0.050341332952181496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,7,0.06321600079536438
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,7,0.050069332122802734
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,31,0.06428800026575725
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,63,0.06571733454863231
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,127,0.19614932934443155
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,127,0.05348266661167145
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,255,0.3789653380711873
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,31,0.050623998045921326
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,63,0.05121066669623057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,255,0.19794134298960367
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,511,0.8185439904530843
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,511,0.40538668632507324
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1,0.11219732960065205
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1,0.08999466896057129
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,3,0.09005866448084514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1023,1.617733319600423
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,3,0.11417599519093831
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,1,128,1,float16,fp8,1023,0.7871200243631998
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,7,0.1167093316713969
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,15,0.11611200372378032
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,7,0.08998933434486389
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,15,0.0902933379014333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,31,0.11787199974060059
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,63,0.19353600343068442
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,127,0.3770506779352824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,31,0.09078933795293172
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,63,0.09171733260154724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,127,0.20374399423599243
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,3,0.008890666688481966
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,3,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,7,0.008837333569924036
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,255,0.8133546511332194
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,7,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1,0.010874666273593903
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,15,0.008869333192706108
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,31,0.008821333448092142
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,31,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,63,0.00890666681031386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,63,0.010885333021481832
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,127,0.009082666908701261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,127,0.01109333336353302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,255,0.012330666184425354
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,255,0.3869813283284505
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,255,0.011994666109482447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,511,0.012805332740147909
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,511,0.013450667262077332
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1023,0.01381333296497663
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,2047,0.02033599962790807
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,2047,0.022309333086013794
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,4095,0.025008000433444977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,4095,0.03416533271471659
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,8191,0.03448000053564707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,8191,0.058090666929880776
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,16383,0.10590400298436482
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,16383,0.19234667221705118
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,15,0.011125333607196808
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,32767,0.37221332391103107
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,32767,0.21401600042978922
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,65535,0.730784018834432
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,65535,0.4142186641693115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1,0.007733333234985669
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,1023,0.016496000190575916
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,131071,1.6292373339335124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,1,128,1,float16,fp8,131071,0.8150560061136881
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,3,0.007706666365265846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,7,0.007850666840871176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,3,0.011114666859308878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,7,0.01098666712641716
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,15,0.007877333089709282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,15,0.010970667004585266
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,31,0.011050666371981302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,63,0.008005333443482717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,31,0.008005333443482717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,63,0.01101333275437355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,127,0.008309333274761835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,127,0.011306667079528173
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,255,0.010608000059922537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,255,0.011957333733638128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,511,0.015194666882356008
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,511,0.013610667238632837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,1023,0.016623999923467636
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1023,0.024703999360402424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,2047,0.04345066845417023
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,2047,0.022639999787012737
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,4095,0.08067200084527333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,4095,0.03475199888149897
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,16383,0.3641440073649089
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,8191,0.1843199928601583
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,16383,0.19170665740966797
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,8191,0.05849599838256836
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,32767,0.36974934736887616
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,32767,0.7221226692199707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,65535,0.8034400145212809
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,65535,1.6031732559204102
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1,0.007791999727487564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,511,0.7929066816965739
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,3,0.007861333588759104
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,3,0.010960000256697336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,511,1.6076693534851074
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,2,2,128,1,float16,fp8,131071,1.6085012753804524
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,7,0.007823999971151352
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,131071,3.2034079233805337
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,15,0.007781333600481351
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,15,0.01101333275437355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,63,0.007936000203092894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,63,0.011007999380429586
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,31,0.010954666882753372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,127,0.008383999889095625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,127,0.01116266722480456
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,255,0.012709333250919977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,255,0.01062400018175443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,511,0.01525866612792015
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,511,0.015664000064134598
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,1023,0.021664001047611237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1023,0.024485332270463307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,2047,0.043136000633239746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,2047,0.0336053321758906
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,4095,0.08061333497365315
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,4095,0.057573333382606506
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,8191,0.18641066551208496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,16383,0.21271467208862305
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,8191,0.10517332951227824
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,16383,0.3667680025100708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,32767,0.7291306654612223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,32767,0.41420801480611164
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,7,0.01099733387430509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,65535,0.8336586952209473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,131071,3.2272586822509766
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,31,0.007936000203092894
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1,0.007418666655818622
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,32,1,1,1,128,1,float16,fp8,131071,1.6638773282368977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,7,0.0074506668994824094
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,3,0.010458666831254959
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,3,0.007642666498819987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,15,0.007536000261704127
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,15,0.010480000327030817
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,65535,1.614496072133382
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,31,0.007781333600481351
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,31,0.010703999549150467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,63,0.00761600024998188
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,63,0.010485333700974783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,7,0.010549332946538925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,255,0.011637333780527115
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,127,0.01080000028014183
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,255,0.010309333602587381
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,511,0.01471466695268949
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,511,0.011642667154471079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1023,0.024432001014550526
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,1023,0.011546666423479715
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,2047,0.04294399917125702
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,2047,0.011514666179815928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,4095,0.012058666596810022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,4095,0.08009600142637889
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,8191,0.1549493372440338
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,8191,0.01357866699496905
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,127,0.007920000081261
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,16383,0.30356266101201373
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,16383,0.016501333564519882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,32767,0.02274133265018463
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,65535,0.03335466732581457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1,1,1,1,128,1,float16,fp8,131071,0.05552533268928528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1,0.007621333623925845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1,0.010816000401973724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,32767,0.6023199955622355
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,3,0.007600000128149986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,65535,1.2005013624827068
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,3,0.010768000036478043
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,7,0.007621333623925845
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,15,0.007706666365265846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,7,0.010879999647537867
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,31,0.007770666852593422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,15,0.010842667271693548
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,31,0.0107893335322539
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,63,0.007791999727487564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,131071,2.3922932942708335
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,63,0.010837333897749582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,127,0.0080960001796484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,127,0.01099733387430509
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,255,0.010288000106811523
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,255,0.011690666278203329
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,511,0.014655999839305878
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,511,0.011626667032639185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1023,0.024298667907714844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,1023,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,2047,0.04287999868392944
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,2047,0.012069333344697952
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,4095,0.0801386684179306
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,4095,0.013455999394257864
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,8191,0.016597333053747814
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,8191,0.15501333276430765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,16383,0.022554665803909302
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,16383,0.3043786684672038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1023,3.205882708231608
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,32767,0.03454933315515518
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,32767,0.6024640003840128
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,65535,0.056986664732297264
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2,1,1,1,128,1,float16,fp8,131071,0.10174933075904846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,65535,1.2005706628163655
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1,0.011098666737476984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,3,0.007983999947706858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1,0.007813333223263422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,3,0.011226666470368704
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,131071,2.434271971384684
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,15,0.00795199970404307
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,7,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,31,0.007967999825874964
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,15,0.011109333485364914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,31,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,63,0.01119999960064888
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,63,0.008074666683872541
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,127,0.00843733362853527
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,127,0.011648000528415045
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,255,0.010826667149861654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,255,0.013114667187134424
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,2,2,128,1,float16,fp8,1023,1.5567253430684407
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,511,0.016229332735141117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,511,0.015530666957298914
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,1023,0.02216533323129018
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1023,0.025045332809289295
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,2047,0.03429333368937174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,2047,0.043866669138272606
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,8191,0.3641066551208496
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,8191,0.19172799587249756
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,4095,0.186298668384552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,16383,0.7197066942850748
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,32767,1.5943412780761719
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,16383,0.36994131406148273
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1,0.007642666498819987
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1,0.01121066634853681
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,32767,0.8028266429901123
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,3,0.007914666707317034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,7,0.007893333211541176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,7,0.011146667102972666
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,3,0.01110400011142095
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,65535,1.6207680702209473
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,15,0.007754666730761528
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,31,0.007786666974425316
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,15,0.010992000500361124
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,63,0.007941333577036858
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,63,0.010954666882753372
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,127,0.008026666939258575
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,127,0.011039999624093374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,255,0.010304000228643417
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,255,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,511,0.014970666418472925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,511,0.01173866664369901
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,65535,2.8579734166463218
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,31,0.010837333897749582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1023,0.024341332415739696
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,1023,0.011941333611806234
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,131071,3.216634750366211
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,2047,0.013381333400805792
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,4095,0.01640533283352852
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,2047,0.04312000175317129
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,4095,0.08036800225575765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,131071,6.401135762532552
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,8191,0.022672000030676525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,16383,0.3041759928067525
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,32767,0.6029119888941447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,16383,0.034661332766215004
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,65535,0.10597333312034607
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,65535,1.2205440203348796
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,131071,0.2135253349939982
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,131071,2.436901410420736
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1,0.008042666440208754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1,0.011354666203260422
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,3,0.008127999802430471
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,4,1,1,1,128,1,float16,fp8,32767,0.058287998040517174
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,7,0.008282666405042013
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,8191,0.1551359991232554
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,3,0.011503999431928
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,7,0.008207999790708223
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,7,0.011445333560307821
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,15,0.008192000289758047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,15,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,31,0.008341333518425623
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,63,0.008527999743819237
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,63,0.011744000017642975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,31,0.011541333049535751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,127,0.009045333291093508
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,127,0.012144000579913458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,255,0.013786666095256805
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,255,0.011498666057984034
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,64,1,1,1,128,1,float16,fp8,4095,0.058378666639328
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,511,0.016927999754746754
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,511,0.016250666230916977
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,1023,0.02333866556485494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1023,0.02644266684850057
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,2047,0.18702399730682373
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,2047,0.035546667873859406
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,8191,0.7224213282267252
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,8191,0.3691840171813965
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,4095,0.19215466578801474
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,4095,0.3654400110244751
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,16383,1.4357333183288574
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,16383,0.7237066427866617
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,3,0.011893333246310553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,32767,1.4366772969563801
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,32767,2.8599093755086265
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1,0.011973333855470022
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1,0.013541333377361298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,3,0.013610667238632837
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,7,0.012128000458081564
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,7,0.01369599997997284
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,15,0.012085333466529846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,15,0.013541333377361298
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,31,0.012165332833925882
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,128,1,1,1,128,1,float16,fp8,65535,3.1318880716959634
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,31,0.013749333719412485
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,65535,6.429679870605469
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,63,0.012543999900420507
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,63,0.014245333770910898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,127,0.013568000247081121
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,127,0.014869333555301031
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,255,0.01782400036851565
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,511,0.022970666488011677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1023,0.18720000982284546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,1023,0.03315199911594391
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,255,0.01850133389234543
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,511,0.02849599967400233
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,2047,0.3669546842575073
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,2047,0.19339199860890707
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,8191,1.4360586802164714
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,8191,0.7274773120880127
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,4095,0.3714933395385742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,4095,0.8115573724110922
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1,0.020101333657900494
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1,0.019205333044131596
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,3,0.020010666300853092
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,3,0.01945066700379054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,256,1,1,1,128,1,float16,fp8,16383,1.437071959177653
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,16383,3.019536018371582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,15,0.019637333850065868
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,15,0.020090666910012562
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,31,0.019962667177120846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,31,0.02013333390156428
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,7,0.01960533360640208
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,7,0.020047999918460846
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,63,0.020303999384244282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,63,0.020517333100239437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,127,0.022117334107557934
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,127,0.021733333667119343
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,255,0.032186667124430336
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,255,0.02677333354949951
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,511,0.18778133392333984
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,511,0.036943999429543815
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,1023,0.20829866329828897
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1023,0.3859573205312093
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1,0.0075573331365982694
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,8191,2.859312057495117
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,3,0.007807999849319458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,7,0.007807999849319458
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,3,0.010869332899649939
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,2047,0.7480160395304362
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,7,0.010901333143313726
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,15,0.00784533346692721
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,15,0.010784000158309937
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,31,0.010911999891201654
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,63,0.008005333443482717
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,63,0.010847999403874079
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,2047,0.37404799461364746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,127,0.008063999935984612
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,127,0.011039999624093374
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,8191,1.4372159639994304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,255,0.010442666709423065
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,511,0.015034666905800501
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,511,0.011834666132926941
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1023,0.02438933402299881
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,2047,0.043023998538653054
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,2047,0.016415999581416447
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,4095,0.08042666812737782
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,4095,0.022341333329677582
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,8191,0.1551040013631185
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,16383,0.05815466741720835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,8191,0.0342399999499321
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,16383,0.3043786684672038
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,32767,0.10575466354688008
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,32767,0.6152480045954386
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,1023,0.013477332890033722
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,65535,0.2128373384475708
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,65535,1.2239573001861572
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,131071,0.42629865805308026
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,131071,2.4374186197916665
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,31,0.00903466654320558
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,8,1,1,1,128,1,float16,fp8,255,0.011770666887362799
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,4095,1.613525390625
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,512,1,1,1,128,1,float16,fp8,4095,0.7848693529764811
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1,0.03347733368476232
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1,0.029445332785447437
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,3,0.03425066669782003
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,3,0.029440000653266907
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,7,0.034815999368826546
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,31,0.03497066597143809
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,7,0.029552000264326733
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,31,0.029487999776999157
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,63,0.029450667401154835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,63,0.03579200059175491
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,127,0.03828799972931544
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,15,0.034789333740870156
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,15,0.02959999938805898
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,127,0.03038399914900462
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,255,0.20937599738438925
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,255,0.040949332217375435
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,511,0.37009068330128986
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,511,0.19337600469589233
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1023,0.7268426418304443
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,1023,0.374234676361084
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,2047,1.6154613494873047
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,2047,0.7287946542104086
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1,0.05995733539263407
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1,0.0499893327554067
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,3,0.06084266801675161
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,3,0.05016533533732096
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,1024,1,1,1,128,1,float16,fp8,4095,1.55186128616333
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,4095,3.227882703145345
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,15,0.06168533364931742
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,15,0.050106664498647056
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,31,0.06352533400058746
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,7,0.062090665102005005
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,7,0.05002133548259735
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,31,0.05032533407211304
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,127,0.19184533754984537
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,255,0.37357866764068604
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,127,0.0532533327738444
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,255,0.19733333587646484
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,63,0.050800000627835594
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,63,0.06342400113741557
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1,0.007818666597207388
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,3,0.007760000104705493
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1,0.01090666651725769
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,3,0.011055999745925268
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,7,0.007850666840871176
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,7,0.010821333775917688
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,15,0.007765333478649457
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,15,0.01081066702802976
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,31,0.007834666719039282
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,31,0.010805333654085795
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,63,0.00789866658548514
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,63,0.010816000401973724
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,127,0.011205332974592844
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,127,0.008143999924262365
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,255,0.010485333700974783
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,255,0.011850666254758835
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,511,0.015040000279744467
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,511,0.013573333621025085
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1023,0.02462399999300639
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,1023,0.016314666718244553
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,2047,0.04305600126584371
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,2047,0.02236266682545344
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,4095,0.08038933575153351
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,4095,0.034261333445707955
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,511,0.7299359639485677
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,8191,0.15501333276430765
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,8191,0.05813866853713989
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,16383,0.3122719923655192
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,32767,0.6181066830952963
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,32767,0.2126880089441935
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,65535,0.4140746593475342
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,511,0.37829331556955975
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,65535,1.2262132962544758
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,131071,0.8338560263315836
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,131071,2.4488800366719565
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,16,1,1,1,128,1,float16,fp8,16383,0.10602666934331258
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1023,1.4402880668640137
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,1023,0.7330933411916097
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flashinfer,2048,1,1,1,128,1,float16,fp8,2047,1.5498026212056477
VLLM,0.14.0,NVIDIA L40S,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,2047,3.218496004740397
