framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1,0.013712000101804733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,3,0.013728000223636627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,7,0.013749333719412485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,15,0.013631999492645264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,31,0.013674666484196981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,63,0.013839999834696451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,127,0.013882666826248169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,255,0.021759999295075733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1023,0.030789333085219067
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,2047,0.049285332361857094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,4095,0.06961599985758464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,3,0.020495999604463577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,511,0.0266239990790685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,15,0.020725333442290623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,31,0.020586666961510975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,127,0.02094399929046631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,255,0.029365333418051403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1023,0.05157866577307383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,4095,0.11577066779136658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,3,0.02311466634273529
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,7,0.02333866556485494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,15,0.02316266546646754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,31,0.02327999969323476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,63,0.0235359991590182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,127,0.024010665714740753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,255,0.028309332827727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,511,0.053871999184290566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1023,0.07387199997901917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,2047,0.11784533659617107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,4095,0.19459199905395508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1,0.027248000105222065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,3,0.027248000105222065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,7,0.027717334528764088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,15,0.027653334041436512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,31,0.027701333165168762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,63,0.027829334139823914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,127,0.028517333169778187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,255,0.04623466730117798
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,511,0.07322666545708974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1023,0.12422933181126912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,2047,0.2241599957148234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,4095,0.42110931873321533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,3,0.01268799975514412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,7,0.012768000364303589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,15,0.012890666723251343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,31,0.012768000364303589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,63,0.012847999731699625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1,0.02067733307679494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,127,0.012842666357755661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,255,0.018650667121013004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,511,0.01912533367673556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1023,0.02021866664290428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,2047,0.02223466585079829
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,4095,0.025989333788553875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,8191,0.03259200106064478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,16383,0.05134400228659312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1,0.012293333808581034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,3,0.012469333906968435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,7,0.012762666990359625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,15,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,31,0.012634667257467905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,63,0.012608000387748083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,127,0.012655999511480331
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,255,0.01829333355029424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,511,0.01886933296918869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1023,0.02004266654451688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,2047,0.02186133215824763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,4095,0.026047999660174053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,8191,0.035536001125971474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,16383,0.04050133377313614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1,0.012533333152532578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,7,0.01257066677014033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,3,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,15,0.012938667088747025
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,31,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,63,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,127,0.012826666235923767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,255,0.018330667167901993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,511,0.018816000471512478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,2047,0.022085333863894146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1023,0.020197333147128422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,8191,0.033861334125200905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,4095,0.02897600084543228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,16383,0.047872001926104225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1,0.012634667257467905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,7,0.012762666990359625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,3,0.012576000144084295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,15,0.012602667013804117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,31,0.012842666357755661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,63,0.012703999876976013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,127,0.01268799975514412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,255,0.018570666511853535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1023,0.020096000283956528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,511,0.01877333347996076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,7,0.020794666061798733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,2047,0.025429333249727886
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,4095,0.02994133283694585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,8191,0.044026667873064675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1,0.012736000120639801
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,16383,0.06472533444563548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,3,0.012810666114091873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,7,0.012762666990359625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,15,0.012965332716703415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,31,0.012853333105643591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,63,0.01302933320403099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,127,0.012863999853531519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,255,0.018837332725524902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,511,0.019285333653291065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1023,0.020517333100239437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,2047,0.022677332162857056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,4095,0.026416001220544178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,8191,0.037530665596326195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,16383,0.04228800038496653
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1,0.0183999997874101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,3,0.01850133389234543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,7,0.018624000251293182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,15,0.01841066653529803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,31,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,63,0.01848000039656957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,127,0.018405333161354065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,255,0.0222080002228419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,511,0.023007998863856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1023,0.023978665471076965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,4095,0.03332266708215078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,2047,0.02626666675011317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,8191,0.03761066744724909
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,16383,0.056314667065938316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1,0.018650667121013004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,3,0.018474667022625606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,7,0.018629333625237148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,15,0.01889066646496455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,31,0.018511999398469925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,63,0.018687999496857326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,127,0.018602666755517323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,255,0.02221333235502243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,511,0.022970666488011677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,2047,0.029114666084448498
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1023,0.02420799930890401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,4095,0.03379199902216593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,8191,0.04642133414745331
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,16383,0.07210666437943776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1,0.018330667167901993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,3,0.018415999909241993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,15,0.018629333625237148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,7,0.018976000448067982
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,31,0.01860800012946129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,127,0.018645333747069042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,511,0.023082666099071503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,63,0.018725333114465077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,255,0.022469334304332733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,63,0.020949333906173706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1023,0.02731200059254964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,2047,0.03176533430814743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,4095,0.04832000037034353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,8191,0.06980266670385997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,16383,0.10869333148002625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1,0.014101333916187286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,3,0.013989333063364029
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,7,0.01414399966597557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,15,0.014074667046467463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,511,0.034501334031422935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,31,0.013866666704416275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,2047,0.07160000006357829
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,63,0.014186666657527288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,127,0.014511999984582266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,255,0.03035733352104823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,511,0.03499199946721395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1,0.02518933266401291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,3,0.025285333395004272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,2047,0.07482666770617168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1023,0.05491200089454651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1,0.023290666441122692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,7,0.025114665428797405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,15,0.025253333151340485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,31,0.025274666647116344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,63,0.025616000096003216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,127,0.025914666553338368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,511,0.06359466910362244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1,0.029472000896930695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,255,0.03090133269627889
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,3,0.029461334149042766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1023,0.08444266517957051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,7,0.029834667841593426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,15,0.02977066735426585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,2047,0.1313973367214203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,63,0.03011200080315272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,31,0.029792000850041706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,127,0.03108799954255422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,511,0.07653866708278656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,255,0.0491839994986852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,2047,0.2270666758219401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,3,0.03601066768169403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,7,0.03612266729275385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,15,0.0365226666132609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1023,0.12501866618792215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,63,0.037018666664759316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1,0.0360959991812706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,31,0.03618666778008143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,127,0.048384000857671104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1023,0.1958293318748474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1,0.012863999853531519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,3,0.012986666212479273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,255,0.07090666890144348
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,7,0.013023999830087027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,63,0.013125333935022354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,31,0.012960000584522883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,15,0.013130666067202887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,255,0.018826667219400406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,2047,0.35736000537872314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1023,0.02083733429511388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,511,0.01958400011062622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,511,0.11306666334470113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,127,0.013199999928474426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,2047,0.02348800003528595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,4095,0.030400000512599945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1,0.01882133384545644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,16383,0.05332799752553304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,7,0.019205333044131596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,8191,0.03525333354870478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,15,0.019029332945744198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,31,0.019226666539907455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,63,0.01905599981546402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,127,0.01926933353145917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,255,0.022698665658632915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,511,0.02314666658639908
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1023,0.02458133300145467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,4095,0.03446933378775915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,2047,0.029829333225886028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,3,0.019178666174411774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,8191,0.05107733110586802
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,16383,0.07274666428565979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1,0.01941866676012675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,3,0.01931200052301089
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,31,0.019226666539907455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,7,0.0191040001809597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,63,0.019365333020687103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,15,0.01940800001223882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,127,0.019546666493018467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,255,0.02274133265018463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,511,0.02346666653951009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,2047,0.03260799994071325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1023,0.02777066578467687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,4095,0.045797333121299744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,8191,0.0680266668399175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,16383,0.10859200358390808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,3,0.019461333751678467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,7,0.019333332777023315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,31,0.019653332730134327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1,0.019343999524911244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,15,0.019482667247454327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,63,0.019472000499566395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,127,0.019280000279347103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,255,0.023039999107519787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,511,0.026911998788515728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1023,0.031221332649389904
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,4095,0.0669706662495931
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,8191,0.10763733585675557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,2047,0.045642669002215065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,16383,0.1879840095837911
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1,0.017743999759356182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,3,0.01748266691962878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,63,0.018346666047970455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,31,0.017610666652520496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,15,0.017781333376963932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,7,0.017898666361967724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,127,0.018239999810854595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,255,0.02348800003528595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,511,0.06934933364391327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1,0.034234667817751564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,3,0.03442133218050003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,7,0.034671999514102936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,15,0.03482666611671448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,31,0.03468266626199087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1023,0.08754133184750874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,63,0.03523733218510946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,127,0.03746666759252548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1,0.0407679999868075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,255,0.057205334305763245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,3,0.0408746674656868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,511,0.08319999774297078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1023,0.13562132914861044
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,15,0.041375999649365745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,63,0.043418665726979576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,31,0.04154666761557261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,7,0.04121066629886627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,127,0.053258667389551796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1,0.056474665800730385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,511,0.12193600336710612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,255,0.07729599873224895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,7,0.05752533177534739
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,15,0.05759466687838236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,3,0.05667733152707418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,31,0.05991466840108236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1023,0.20662933588027954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,63,0.06493866443634033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,127,0.07871999839941661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,255,0.12428800264994304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1023,0.35121599833170575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,1,0.02383466561635335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,511,0.2002026637395223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,3,0.023978665471076965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,15,0.024720000723997753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,7,0.024266667664051056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,31,0.024357333779335022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,127,0.029066666960716248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,63,0.02516799916823705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,1,0.048725331823031105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,255,0.050245334704717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,7,0.04937066634496053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,3,0.049312000473340355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,15,0.05019199848175049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,31,0.051669334371884666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,63,0.05781333148479462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,127,0.06533333162466685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,1,0.06605866551399231
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,3,0.06659733255704244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,255,0.09030399719874065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,7,0.06800533334414165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,15,0.06980800131956737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,31,0.07333866755167644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,63,0.07760000228881836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,127,0.0897706647713979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,1,0.09704533219337463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,255,0.1363146702448527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,7,0.10308266679445903
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,15,0.10593600074450175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,3,0.09833066662152608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,31,0.10653332869211833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,63,0.11015466849009196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,127,0.1323253313700358
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,255,0.2177706758181254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,1,0.03612266729275385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,3,0.03669866671164831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,7,0.03829866647720337
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,15,0.03984000037113825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,31,0.0420959989229838
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,63,0.04704533517360687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,127,0.05451733370621999
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,1,0.09167466560999553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,3,0.09269866347312927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,7,0.0942026674747467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,15,0.09478933612505595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,31,0.09710400303204854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,63,0.10063999891281128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,127,0.11453866958618164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,1,0.1252906620502472
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,3,0.12805333733558655
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,7,0.1300320029258728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,15,0.12997333208719888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,31,0.12999999523162842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,63,0.13435733318328857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,127,0.15966399510701498
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,1,0.18698134024937949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1,0.013023999830087027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,3,0.1903839906056722
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,3,0.013162666310866674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,7,0.013130666067202887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,31,0.012944000462690989
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,15,0.013317332913478216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,63,0.013210666676362356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,511,0.020101333657900494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,255,0.019498666127522785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,7,0.19468265771865845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1023,0.021776000658671062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,15,0.19338667392730713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,2047,0.02757333219051361
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,16383,0.07210666437943776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,4095,0.03219199925661087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1,0.019626667102177937
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,7,0.019365333020687103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,8191,0.050586665670077004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,31,0.01961600035429001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,31,0.19268800814946493
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,63,0.019519999623298645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,15,0.01937066639463107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,63,0.19602133830388388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,3,0.019567999988794327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,255,0.023306667804718018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,127,0.019882666567961376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1023,0.028832000990708668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,2047,0.03297599901755651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,4095,0.05272000034650167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1,0.019904000063737232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,16383,0.11583999792734782
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,3,0.019567999988794327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,511,0.024298667907714844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,31,0.020037333170572918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,127,0.24385066827138266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,15,0.019797333826621372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,7,0.019573333362738293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,127,0.019685332973798115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,63,0.01982933282852173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,255,0.023578666150569916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,511,0.02734400083621343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,4095,0.07055999835332234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1023,0.03200533241033554
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,8191,0.10838933785756429
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,16383,0.1925599972407023
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,8191,0.07351999978224437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1,0.0198186660806338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,3,0.02014933278163274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,7,0.019727999965349834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,31,0.019808000574509304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,15,0.0199946661790212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,63,0.0201706662774086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,127,0.0201706662774086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,255,0.026858667532602947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1023,0.04331199824810028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,511,0.03146666785081228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,2047,0.06779199838638306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,4095,0.10973333319028218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,8191,0.18796267112096152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,16383,0.3449866771697998
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,2047,0.049413333336512245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,1,0.06966933111349742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,3,0.06891199946403503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,7,0.07037333150704701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,15,0.07162133355935414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,1,0.17164266109466553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,31,0.07457066575686137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,63,0.08186133205890656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,3,0.17085866133371988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,7,0.17300800482432047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,1,0.23831466833750406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,31,0.17348267634709677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,15,0.17322667439778647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,63,0.17899733781814575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,3,0.2369813323020935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,7,0.2405280073483785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,15,0.24022932847340903
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,31,0.2403786579767863
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,1,0.35949865976969403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,63,0.24766399463017783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,3,0.3598613341649373
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,7,0.3656533161799113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,15,0.3630186716715495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,31,0.3637973467508952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,63,0.3773546616236369
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,1,0.13359999656677246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,3,0.13589866956075033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,7,0.1349120040734609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,31,0.1383093297481537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,15,0.1378666659196218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,1,0.3269919951756795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,3,0.3274293343226115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,7,0.32706133524576825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,1,0.45613332589467365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,15,0.3264213403065999
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,31,0.3231733242670695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,3,0.45602667331695557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,7,0.462010661760966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,15,0.4641546805699666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1,0.01333333303531011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,7,0.013370666652917862
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,3,0.01331199953953425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,31,0.013114667187134424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,31,0.4630346695582072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,63,0.013541333377361298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,127,0.013647999614477158
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,511,0.021040000021457672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1023,0.026176000634829204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,255,0.02045866722861926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,2047,0.030799999833106995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,4095,0.04789333542188009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,8191,0.07060266534487407
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1,0.01989866668979327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,3,0.019904000063737232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,7,0.02011200040578842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,15,0.020165332903464634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,31,0.020074666788180668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,63,0.020090666910012562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,127,0.020031999796628952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,255,0.024090667565663654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,511,0.028384000062942505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1023,0.033045334120591484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,1,0.72270401318868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,4095,0.07147199908892314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,2047,0.050527999798456825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,8191,0.11533332864443462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,3,0.020213333268960316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,15,0.020266667008399963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,7,0.020117333779732387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1,0.020058666666348774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,31,0.02027733375628789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,63,0.020207999895016353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,127,0.020442667106787365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,255,0.02757866680622101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,511,0.03230399886767069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,2047,0.07123733560244243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1023,0.04791999856630961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,4095,0.1129919985930125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1,0.022495999932289124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,8191,0.19434666633605957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,3,0.022704000274340313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,7,0.022885332504908245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,15,0.022837333381175995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,31,0.02306666721900304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,63,0.023215999205907185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,127,0.02346133440732956
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,255,0.028218666712443035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,511,0.050186668833096824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1023,0.07022400200366974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,2047,0.11582400401433308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,4095,0.19077332814534506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1,0.013338666409254074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,8191,0.3466453154881795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,3,0.013455999394257864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,7,0.013482666263977686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,15,0.013642666240533194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,31,0.013461332768201828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,63,0.013471999516089758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,127,0.013717333475748697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,255,0.020869334538777668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,511,0.02516799916823705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1023,0.029722665747006733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,2047,0.04663999875386556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,4095,0.06842133402824402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1,0.020213333268960316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,3,0.02015999952952067
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,15,0.02038399999340375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,7,0.020245333512624104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,31,0.02029866725206375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,127,0.020549333343903225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,255,0.0283146674434344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,63,0.020421333611011505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,511,0.032618666688601174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1023,0.04987733562787374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,4095,0.12095999717712402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,2047,0.0713973343372345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,7,0.7077120145161947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1,0.022805333137512207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,3,0.022917332748572033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,31,0.023205332458019257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,7,0.023178666830062866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,15,0.023082666099071503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,63,0.023408000667889912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,255,0.028384000062942505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,2047,0.11709333459536235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,127,0.023914667467276256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1023,0.07311466832955678
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,511,0.05235200126965841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,4095,0.195306658744812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1,0.027029333015282948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,3,0.02720533311367035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,31,0.027471999327341717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,7,0.027461332579453785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,15,0.027717334528764088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,63,0.0277813325325648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,127,0.028592000404993694
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,255,0.04484266539414724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1023,0.129013329744339
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,2047,0.22285334269205728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,511,0.07311466832955678
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,3,0.012768000364303589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,7,0.012746666868527731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,31,0.012608000387748083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,15,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,4095,0.41997865835825604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,63,0.012778667112191519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,127,0.012805332740147909
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,255,0.018245333184798557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,511,0.018725333114465077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1023,0.019882666567961376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,8191,0.03177600105603536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,2047,0.02162666618824005
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,4095,0.025333332518736523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,16383,0.04875733455022176
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1,0.012298667182525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,3,0.012373333175977072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,7,0.012709333250919977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,31,0.01257066677014033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,15,0.012762666990359625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,127,0.012629333883523941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,63,0.012560000022252401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,255,0.018266666680574417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,511,0.018464000274737675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1023,0.019573333362738293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,2047,0.021344001094500225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,4095,0.025360000630219776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,16383,0.03915733347336451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,8191,0.03501333296298981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,3,0.012362666428089142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,7,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,15,0.012639999389648438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,31,0.012821332861979803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,63,0.012655999511480331
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,127,0.01251199965675672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,511,0.01851733277241389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,3,0.7023359934488932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,255,0.01826133330663045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1023,0.019738666713237762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,2047,0.021583999196688335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,4095,0.028565332293510437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,8191,0.03339199970165888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,16383,0.046575998266537987
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1,0.012741333494583765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,3,0.012671999633312225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,7,0.012703999876976013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,15,0.012741333494583765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,31,0.012778667112191519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,63,0.012815999488035837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,127,0.012741333494583765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,255,0.01815466706951459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,511,0.01886933296918869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,2047,0.0249439999461174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,4095,0.02956266701221466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,8191,0.040549332896868386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,16383,0.06396799782911937
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,31,0.7178026835123698
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,7,0.01259200026591619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1,0.012736000120639801
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,3,0.01257066677014033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,15,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1023,0.020245333512624104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,63,0.012597333639860153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,127,0.012810666114091873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,255,0.018687999496857326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,511,0.01932799940307935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1023,0.020197333147128422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,2047,0.022096000611782074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,8191,0.03688533355792364
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,4095,0.02601066728432973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,16383,0.04148799926042557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1,0.018725333114465077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,3,0.01846933364868164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,7,0.018645333747069042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,15,0.018768000106016796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,63,0.018661333868900936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,127,0.018320000420014065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,255,0.022282667458057404
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,31,0.018789333601792652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,511,0.02277333289384842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1023,0.023728000621000927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,2047,0.025968000292778015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,4095,0.033071999748547874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,16383,0.053871999184290566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,31,0.012714666624863943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,8191,0.03793599953254064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,3,0.0184906671444575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,7,0.01876266673207283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,15,0.018709332992633183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,31,0.018650667121013004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,127,0.018565333137909572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,511,0.022789334257443745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,255,0.022298666338125866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1023,0.023930666347344715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,63,0.018906666586796444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,2047,0.02905600021282832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,15,0.7099359830220541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,4095,0.033759998778502144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,8191,0.048063998421033226
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,15,0.018751999984184902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1,0.01850133389234543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,7,0.01877333347996076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,3,0.01884799947341283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,16383,0.07066666583220164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,31,0.018672000616788864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,63,0.01868266612291336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,127,0.018565333137909572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,255,0.022645334402720135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1023,0.027242665489514668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,511,0.023525332411130268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,4095,0.04450133442878723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,8191,0.06637333333492279
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,16383,0.10890133182207744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,2047,0.032069332897663116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1,0.013834666460752487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,3,0.01394133393963178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,15,0.014117332796255747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,7,0.01403733342885971
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,31,0.014042666802803675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,63,0.014032000054915747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,127,0.014373333503802618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,511,0.031680000325044
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,255,0.026880001028378803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,7,0.023904000719388325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,3,0.023685333629449207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1,0.023813332120577495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,15,0.023797333240509033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,2047,0.06962666908899943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1023,0.04984533290068308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,63,0.02409599969784419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,31,0.024010665714740753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,127,0.02459733436505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,255,0.029071999092896778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1,0.028010666370391846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,511,0.05728533367315928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,3,0.02805333336194356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,7,0.02850666642189026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,2047,0.1269493301709493
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,15,0.028389332195123036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1023,0.0765173335870107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,31,0.028357334434986115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,63,0.02864533414443334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,127,0.030661332110563915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1023,0.1246453324953715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,255,0.048058668772379555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,511,0.07411199808120728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,2047,0.2258560061454773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,3,0.034485332667827606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1,0.034559999903043113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,7,0.035242666800816856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,31,0.035061334570248924
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,127,0.04590400060017904
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,63,0.03555733213822047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1023,0.19468265771865845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,15,0.035391998787721
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,255,0.06814399858315785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,3,0.012618667135636011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1,0.012794667234023413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,7,0.013104000439246496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,15,0.012847999731699625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,63,0.012784000486135483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,255,0.01868266612291336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,511,0.11210667093594869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,511,0.019333332777023315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,2047,0.35677866141001385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,31,0.012831999609867731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1023,0.02032533288002014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,2047,0.022442666192849476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,16383,0.052442664901415505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,3,0.018751999984184902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,7,0.02063999945918719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1,0.018730666488409042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,4095,0.030053332448005676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,15,0.018858666221300762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,31,0.018645333747069042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,63,0.01878400022784869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,127,0.01922133316596349
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1023,0.024400000770886738
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,255,0.02235200007756551
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,511,0.023130667706330616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,8191,0.03456533451875051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,4095,0.03420799970626831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,2047,0.02978666623433431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,8191,0.05077333251635233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,7,0.019194666296243668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,16383,0.07236266632874806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,3,0.018965333700180054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,15,0.019018666197856266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,31,0.01931200052301089
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,63,0.019359999646743137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,255,0.02271999915440877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,127,0.018981333822011948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,511,0.023306667804718018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,2047,0.03215999901294708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1023,0.027477333943049114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,4095,0.0454720010360082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,8191,0.06748799979686737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1,0.01887999971707662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,3,0.018965333700180054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,16383,0.10972266395886739
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,7,0.019189332922299702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,31,0.01916266605257988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,15,0.01942933350801468
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,63,0.019551999866962433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,127,0.019487999379634857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,255,0.022842665513356526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1023,0.031141333281993866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,2047,0.042538667718569435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,4095,0.06676800052324931
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,511,0.026730666557947796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,8191,0.10664533575375874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,16383,0.18786134322484335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1,0.016773333152135212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,3,0.01695466662446658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,31,0.018698666244745255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,7,0.01711999997496605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,15,0.017157333592573803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,63,0.017642666896184284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,127,0.017583999782800674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,255,0.02219199885924657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1,0.031471999982992806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,3,0.031632001201311745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,7,0.03188266605138779
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1023,0.08057599763075511
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,15,0.031717332700888314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,511,0.06311999758084615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,31,0.03215999901294708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,63,0.03189333279927572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,127,0.03291733314593633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1,0.03789333254098892
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,255,0.05340266724427541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,511,0.07909333209196727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,3,0.0383840004603068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1023,0.13222933808962503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,15,0.038618666430314384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,7,0.03873066604137421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,31,0.03896533449490865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,127,0.04994666576385498
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,63,0.04001600046952566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,255,0.07259733478228252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1,0.05384000142415365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,3,0.05433600147565206
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,511,0.11691733201344807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1023,0.19951466719309488
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,7,0.05471999943256378
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,31,0.05630399783452352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,15,0.05496533215045929
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,63,0.06133866806825002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,255,0.11474666992823283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,511,0.1951520045598348
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1023,0.34650135040283203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,127,0.07170133292675018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,1,0.022474666436513264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,3,0.022543999056021374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,7,0.022815999885400135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,63,0.02325333406527837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,15,0.022991999983787537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,127,0.024698667228221893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,1,0.044693330923716225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,255,0.045978665351867676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,31,0.02274133265018463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,3,0.044768000642458596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,7,0.045312002301216125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,15,0.04535466432571411
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,31,0.0458186666170756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,63,0.049285332361857094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,127,0.058874666690826416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,255,0.08298133313655853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,1,0.061936000982920326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,3,0.06228800117969513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,7,0.06337599953015645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,31,0.06737599770228068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,15,0.06409599880377452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,63,0.07584533095359802
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,1,0.09238933523495992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,127,0.08403733372688293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,255,0.12785067160924277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,3,0.09285866220792134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,7,0.09642133116722107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,15,0.10076799988746643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,31,0.1015786627928416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,63,0.10416533549626668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,127,0.12583466370900473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,255,0.21076265970865884
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,1,0.03136000037193298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,3,0.03276800115903219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,7,0.032357332607110344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,15,0.033514666060606636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,31,0.034976000587145485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,63,0.04460800190766653
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,127,0.05086933573087057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,3,0.07947733501593272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,1,0.0798773318529129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,15,0.08378133177757263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,7,0.0815413345893224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,31,0.08562133709589641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,63,0.0892693301041921
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,127,0.10327466328938802
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,1,0.11131733655929565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,3,0.11507200201352437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,7,0.11892267068227132
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,15,0.1197706659634908
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,31,0.1193386713663737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,63,0.12256000439325969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,127,0.14638933539390564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,1,0.17385600010553995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,3,0.012986666212479273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1,0.012794667234023413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,7,0.013066666821638743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,3,0.1793760061264038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,15,0.012981332838535309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,7,0.18389334281285605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,31,0.013002666334311167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,63,0.013007999708255133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,127,0.013050666699806849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,511,0.019839999576409657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,255,0.01934933289885521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,15,0.18291199207305908
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1023,0.021157334248224895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,2047,0.02661866694688797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,8191,0.04742933313051859
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,4095,0.031290667752424874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1,0.019253333409627277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,3,0.019029332945744198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,7,0.019226666539907455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,16383,0.07241599758466084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,15,0.01950399950146675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,31,0.18341867129007974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,31,0.01950399950146675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,63,0.01972266659140587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,127,0.01932799940307935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,511,0.023669332265853882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,255,0.023152001202106476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,8191,0.06891199946403503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,4095,0.048538664976755776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1023,0.02811199923356374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1,0.019786667078733444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,16383,0.11494400103886922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,63,0.185205340385437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,2047,0.03278933217128118
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,3,0.01924266666173935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,7,0.019733333339293797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,15,0.019637333850065868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,63,0.019717333217461903
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,31,0.019882666567961376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,255,0.02332799881696701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,127,0.24124266703923544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,511,0.0268053337931633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,2047,0.04528533418973287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1023,0.03156266609827677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,4095,0.06892266869544983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,8191,0.10940266648928325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,3,0.01970133309563001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,15,0.01987733319401741
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1,0.019909333437681198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,7,0.019674666225910187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,127,0.01937066639463107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,31,0.019706666469573975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,63,0.019674666225910187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,127,0.02015999952952067
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,255,0.02651199946800868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,511,0.030997333427270252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,2047,0.06644266843795776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1023,0.04654400050640106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,4095,0.10972799857457478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,8191,0.18764267365137735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,16383,0.3428479830423991
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,16383,0.19165867567062378
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,1,0.06211733321348826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,3,0.06363733112812042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,7,0.0642986645301183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,31,0.06708799799283345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,1,0.14920533696810404
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,63,0.07274133463700612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,3,0.1504906713962555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,15,0.06484266618887584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,7,0.15212266643842062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,15,0.15187199910481772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,1,0.21608533461888632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,31,0.1516586641470591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,63,0.15757333238919577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,3,0.21678400039672852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,7,0.22156800826390585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,15,0.21811733643213907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,31,0.21821333964665732
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,1,0.3399413426717122
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,63,0.22469866275787354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,3,0.3407306671142578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,7,0.3455573320388794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,15,0.34295467535654706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,31,0.34670400619506836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,63,0.36826133728027344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,1,0.11929600437482198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,3,0.12013866504033406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,7,0.12045333782831828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,1,0.28294400374094647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,15,0.12138133247693379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,3,0.2831146717071533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,7,0.28866666555404663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,31,0.12156800429026286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,15,0.28592532873153687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,1,0.41465600331624347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,31,0.28377066055933636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,7,0.4206826686859131
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,3,0.4134506781895955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1,0.01333333303531011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,3,0.013248000293970108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,7,0.013408000270525614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,15,0.013445333888133367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,31,0.013221333424250284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,63,0.01340266689658165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,127,0.013440000514189402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,255,0.019823999454577763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,511,0.02060266708334287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1023,0.025311999022960663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,15,0.4241013526916504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,2047,0.029834667841593426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,4095,0.04587199787298838
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,8191,0.06716266771157582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1,0.019861333072185516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,3,0.019551999866962433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,7,0.020069333414236706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,31,0.42265601952870685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,15,0.019925333559513092
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,31,0.01998399943113327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,1,0.6571679910024008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,63,0.02006400004029274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,127,0.020154666155576706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,511,0.02787200113137563
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1023,0.0322080006202062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,2047,0.05207466582457224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,255,0.023936000963052113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,4095,0.06974400083223979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,8191,0.1141919990380605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,3,0.020224000016848247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,7,0.02035733312368393
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1,0.019861333072185516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,15,0.020207999895016353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,31,0.020453333854675293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,63,0.020314666132132213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,127,0.02038399999340375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,255,0.027258666853109997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,511,0.03202133377393087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,2047,0.07012266914049785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,4095,0.11013333002726237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1023,0.046037331223487854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,8191,0.19317332903544107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1,0.022592000663280487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,3,0.022405333817005157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,7,0.022704000274340313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,15,0.022757334013779957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,31,0.022485333184401195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,63,0.023157333334287006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,127,0.02345066765944163
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,255,0.027664000789324444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,511,0.049045334259668984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1023,0.06955199937025706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,2047,0.11506133278210957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,4095,0.19063466787338257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1,0.013253333667914072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,8191,0.3449920018513997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,3,0.013295999417702356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,3,0.6598560015360514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,7,0.01350933313369751
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,15,0.01350933313369751
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,31,0.013290667285521826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,63,0.013455999394257864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,7,0.6705333391825358
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,255,0.02049066623051961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,511,0.024661332368850708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,127,0.013754667093356451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1023,0.029232000311215717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,15,0.6743413607279459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,3,0.020474666108687718
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,7,0.020538666596015293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1,0.02037866661945979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,15,0.02025066688656807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,2047,0.04699199895064036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,31,0.020442667106787365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,63,0.020186666399240494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,4095,0.06658666829268138
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,511,0.0322773332397143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,255,0.027978666126728058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1023,0.050767997900644936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,4095,0.1127306620279948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,2047,0.0699786643187205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1,0.022842665513356526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,127,0.020794666061798733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,3,0.022645334402720135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,7,0.022991999983787537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,31,0.022698665658632915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,15,0.023024000227451324
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,127,0.02351466566324234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,255,0.02815466622511546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,31,0.6773013273874918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,2047,0.1176533301671346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,63,0.02310933421055476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,511,0.051589335004488625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,4095,0.19625065724054971
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1,0.02697066714366277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1023,0.07251200079917908
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,3,0.027263998985290527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,7,0.0272533322374026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,15,0.02740799884001414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,31,0.027637332677841187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,63,0.027701333165168762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,511,0.07258666555086772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1023,0.12277866403261821
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,255,0.0444213350613912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1,0.012367999802033106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,2047,0.2214613358179728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,3,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,63,0.012586666891972223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,4095,0.41974933942159015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,15,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,127,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,31,0.01249066616098086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,255,0.01836266616980235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,511,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1023,0.019808000574509304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,2047,0.021498667697111767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,4095,0.024885334074497223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,7,0.01251199965675672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,8191,0.03211733450492223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1,0.012245333443085352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,16383,0.048826664686203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,3,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,7,0.012522666404644648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,31,0.012565333396196365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,15,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,63,0.012448000411192576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,127,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,127,0.028575999041398365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,255,0.018005333840847015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,511,0.01858666663368543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,2047,0.02186133215824763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,4095,0.02476799984773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1023,0.019653332730134327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,8191,0.03517866631348928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,16383,0.039642666776975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,3,0.012703999876976013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1,0.01268799975514412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,7,0.012981332838535309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,15,0.012768000364303589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,127,0.012602667013804117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,31,0.012693333129088083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,63,0.012762666990359625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,255,0.018144000321626663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,511,0.018735999862353008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,2047,0.021583999196688335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1023,0.01988799994190534
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,4095,0.028442665934562683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1,0.01249066616098086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,16383,0.046495998899141945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,8191,0.03325333446264267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,3,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,7,0.012784000486135483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,15,0.012650666137536367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,31,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,63,0.012778667112191519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,127,0.012661332885424295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,255,0.01812800019979477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,511,0.018661333868900936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1023,0.01985599969824155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,2047,0.025018667181332905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,4095,0.029706666866938274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,8191,0.04141333450873693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1,0.0124746672809124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,16383,0.06423999865849812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,3,0.012719999998807907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,7,0.012703999876976013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,15,0.012661332885424295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,127,0.012826666235923767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,31,0.012821332861979803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,63,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,511,0.019088000059127808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,255,0.01833600054184596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1023,0.02033599962790807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,2047,0.022005334496498108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,4095,0.02568000058333079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1,0.018522666146357853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,8191,0.036687999963760376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,3,0.01860800012946129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,16383,0.040933333337306976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,7,0.01854933301607768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,15,0.018624000251293182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,31,0.01842133328318596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,63,0.01871466636657715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,127,0.01858666663368543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,255,0.022106667359670002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,511,0.022613334159056347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1023,0.023775999744733173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,4095,0.03257066756486893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,2047,0.025781333446502686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,8191,0.037477334340413414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1,0.01859733338157336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,16383,0.05580799778302511
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,3,0.018602666755517323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,7,0.018768000106016796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,15,0.018719999740521114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,63,0.01874133323629697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,31,0.018906666586796444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,127,0.01884799947341283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,511,0.0229066660006841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,255,0.022240000466505688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1023,0.024080000817775726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,4095,0.03400533397992452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,2047,0.02899733434120814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,8191,0.04688533147176107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1,0.018426666657129925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,16383,0.07095466554164886
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,3,0.018538666268189747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,7,0.018698666244745255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,31,0.01869333287080129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,15,0.01882133384545644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,63,0.018746666610240936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,127,0.01863466699918111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1023,0.027280000348885853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,511,0.023029332359631855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,255,0.022554665803909302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,2047,0.03186133255561193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,4095,0.04390400151411692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,8191,0.06593599915504456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,16383,0.10879466931025188
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,3,0.01368533323208491
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1,0.013728000223636627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,7,0.013983999689420065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,31,0.014090667168299357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,15,0.013829333086808523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,63,0.013866666704416275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,127,0.014250667144854864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,255,0.02589333305756251
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1023,0.046767999728520714
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1,0.02294933299223582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,3,0.023269332945346832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,511,0.030415999392668407
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,7,0.02332266668478648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,15,0.02332799881696701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,63,0.02350933353106181
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,31,0.02348266790310542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,127,0.02404266595840454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,255,0.028543998797734577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,2047,0.06734933455785115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,511,0.05608533322811127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1,0.027615999182065327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1023,0.07543999950091045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,3,0.027935999135176342
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,7,0.02812266598145167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,2047,0.12266666690508525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,15,0.02826666583617528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,31,0.02812266598145167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,127,0.029296000798543293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,63,0.02807466685771942
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,255,0.046240001916885376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,511,0.07377066711584727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,2047,0.2258666753768921
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1,0.0341386670867602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1023,0.12471999724706014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,3,0.034074666599432625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,7,0.03472000112136205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,31,0.03469866762558619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,15,0.03456533451875051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,127,0.044394666949907936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,63,0.03585600107908249
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1,0.012517333030700684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1023,0.19421867529551187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,3,0.01268799975514412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,255,0.06781866649786632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,15,0.012709333250919977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,7,0.012703999876976013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,31,0.012997332960367203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,63,0.012805332740147909
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,2047,0.3569066524505615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,255,0.018474667022625606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,511,0.11132267117500305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,511,0.019029332945744198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,127,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,2047,0.022677332162857056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1023,0.02060266708334287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,8191,0.03417599946260452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,4095,0.029578665892283123
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1,0.018565333137909572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,3,0.018837332725524902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,7,0.01897066707412402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,31,0.01869333287080129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,15,0.019082666685183842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,127,0.01887999971707662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,63,0.01905599981546402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,255,0.022495999932289124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,511,0.023103999594847362
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1023,0.02420799930890401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,16383,0.05231999854246775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,4095,0.033743999898433685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,2047,0.029450667401154835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,16383,0.07039466500282288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,8191,0.05197333296140035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1,0.019109333554903667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,3,0.018874666343132656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,7,0.019130667050679524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,15,0.019167999426523846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,63,0.019226666539907455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,31,0.01932799940307935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,255,0.022661333282788593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,511,0.023082666099071503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,127,0.01930133377512296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,2047,0.03196266790231069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1023,0.027488000690937042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,4095,0.04502933224042257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,8191,0.06750933329264323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1,0.018858666221300762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,16383,0.11001066366831462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,15,0.019082666685183842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,3,0.019199999670187633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,31,0.019274666905403137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,63,0.01932266727089882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,7,0.019434666881958645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,127,0.019413333386182785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1023,0.03089066594839096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,511,0.02672533442576726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,255,0.02271999915440877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,2047,0.04212800165017446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,4095,0.06620799998442332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,8191,0.10829333464304607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,16383,0.20035733779271445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1,0.0164533331990242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,15,0.016879999389251072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,7,0.016842667013406754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,3,0.01653333380818367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,31,0.016943999876578648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,63,0.017301333447297413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,127,0.017765333255132038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,255,0.022261333962281544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1,0.030645333230495453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,511,0.060090666015942894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,3,0.030426666140556335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1023,0.07774933179219563
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,7,0.030837332208951313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,31,0.03089066594839096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,15,0.03091199944416682
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,63,0.03120533376932144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,127,0.0322773332397143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,255,0.0517546683549881
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1,0.0372533326347669
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,3,0.037418665985266365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,7,0.0377813329299291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1023,0.1284160017967224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,511,0.07758933305740356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,15,0.038015998899936676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,31,0.038133333126703896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,63,0.03862400104602178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,127,0.04909333089987437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,511,0.11492266257603963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,255,0.07115733126799266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1,0.05340800185998281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1023,0.19732799132665
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,3,0.05352533360322317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,7,0.05389333268006643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,15,0.054245332876841225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,63,0.060736000537872314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,31,0.05515733361244202
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,127,0.07074666519959767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1023,0.34401599566141766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,255,0.11313600341478984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,511,0.19325866301854452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,1,0.021984001000722248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,3,0.021898667017618816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,15,0.022133332987626392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,7,0.022463999688625336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,31,0.022298666338125866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,63,0.022863999009132385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,127,0.024847999215126038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,1,0.042378668983777366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,3,0.04234133164087931
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,7,0.04284800092379252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,15,0.04351999859015147
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,255,0.045834665497144066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,31,0.04371733466784159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,63,0.045935998360315956
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,127,0.056277334690093994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,255,0.08021333316961925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,1,0.05981333553791046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,3,0.060165335734685264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,7,0.06135466694831848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,15,0.061039999127388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,31,0.0646666685740153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,63,0.06924800078074138
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,127,0.08066133161385854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,1,0.09017067154248555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,3,0.09086400270462036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,255,0.12614933649698892
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,7,0.09501333038012187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,15,0.09770133097966512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,31,0.09965866804122925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,127,0.11992533008257548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,255,0.20703466733296713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,7,0.031114667654037476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,1,0.030597334106763203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,15,0.03230933348337809
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,31,0.03245333333810171
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,3,0.030810666580994923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,1,0.07074666519959767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,3,0.07177599767843883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,7,0.07461866736412048
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,15,0.07706133524576823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,31,0.07932266592979431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,63,0.040549332896868386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,127,0.04800533254941305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,1,0.10439466436704
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,63,0.0836906631787618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,3,0.10791466633478801
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,127,0.09669333696365356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,7,0.11237866679827373
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,15,0.11329066753387451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,31,0.11384532848993938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,1,0.1678666671117147
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,63,0.11723732948303223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,127,0.13987732927004495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,3,0.17274133364359537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,7,0.17862399419148764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,3,0.012826666235923767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1,0.012805332740147909
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,15,0.012821332861979803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,7,0.012975999464591345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,15,0.1773973306020101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,31,0.012960000584522883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,63,0.012815999488035837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,127,0.01313599944114685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,31,0.17721066872278848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,511,0.01970133309563001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,255,0.019023999571800232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1023,0.020853333175182343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,2047,0.026416001220544178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,8191,0.047184000412623085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,4095,0.031136001149813335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,16383,0.07087466617425282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1,0.01930133377512296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,3,0.019248000035683315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,7,0.01937599976857503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,63,0.019674666225910187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,15,0.019178666174411774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,31,0.01913600042462349
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,127,0.019237333287795384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,255,0.02293866624434789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,511,0.023599999646345775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,4095,0.04756266872088114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1023,0.02775466690460841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,2047,0.03302400062481562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,8191,0.07190933326880138
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1,0.019386666516462963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,16383,0.11405866344769795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,3,0.01937066639463107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,7,0.019706666469573975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,15,0.01966933285196622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,31,0.01960533360640208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,63,0.019333332777023315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,127,0.019498666127522785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,255,0.023061332603295643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,63,0.18076799313227335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,511,0.02693866689999898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1023,0.03134933362404505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,2047,0.04418133199214935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,4095,0.0694400022427241
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,8191,0.1109386682510376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,16383,0.18979199727376303
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,3,0.01969066634774208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,7,0.019717333217461903
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,15,0.019637333850065868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,31,0.019589333484570186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,127,0.2217173377672831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,63,0.019765333582957584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,127,0.019866666446129482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1,0.019797333826621372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,255,0.02649066597223282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1023,0.043247997760772705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,511,0.031061333914597828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,4095,0.10818666219711304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,8191,0.200762669245402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,16383,0.3429066737492879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,2047,0.06679466863473256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,3,0.05937600135803223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,1,0.05894933144251505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,15,0.06171200176080068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,7,0.060831998785336815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,31,0.06326933205127716
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,63,0.06870933373769124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,1,0.13709867000579834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,1,0.20426666736602783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,3,0.13967466354370117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,15,0.14217600226402283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,7,0.1422826647758484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,63,0.14600533246994019
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,31,0.14127999544143677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,3,0.20780799786249796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,7,0.2092426617940267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,15,0.20853867133458456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,31,0.20919466018676758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,1,0.3282240033149719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,63,0.21439999341964722
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,3,0.330186665058136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,7,0.3371359904607137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,15,0.33346664905548096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,31,0.3340533177057902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,63,0.3454666535059611
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,1,0.11271466811498006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,15,0.11295466621716817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,3,0.11272533734639485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,7,0.11354666948318481
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,31,0.1136853297551473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,1,0.26307199398676556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,3,0.2646720012029012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,7,0.2662293314933777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,15,0.265173335870107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,1,0.39403732617696124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,31,0.26224533716837567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,3,0.394320011138916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,7,0.4010239839553833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,15,0.4025386571884155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1,0.013114667187134424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,3,0.01310933381319046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,7,0.013183999806642532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,15,0.013194666554530462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,31,0.013141332815090815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,63,0.013232000172138214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,127,0.013274667163689932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,255,0.01971199984351794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,511,0.02062400057911873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1023,0.02515200028816859
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,2047,0.02956799914439519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,31,0.4039306640625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,4095,0.04619200030962626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1,0.01964266722400983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,3,0.019776000330845516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,8191,0.06760000189145406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,1,0.6399093468983968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,7,0.019823999454577763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,15,0.019920000185569126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,31,0.01978133370478948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,63,0.01970133309563001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,127,0.019914666811625164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,255,0.02385599911212921
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,511,0.027647999425729115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,3,0.6411199967066447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1023,0.032101333141326904
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,2047,0.0461706668138504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,4095,0.06956266860167186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,8191,0.11341866850852966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,7,0.6484533150990804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1,0.01995733380317688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,3,0.01998399943113327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,7,0.020282667130231857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,31,0.020245333512624104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,15,0.020186666399240494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,127,0.0204373337328434
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,255,0.027162666122118633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,63,0.020080000162124634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,511,0.03152533372243246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1023,0.046581332882245384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,2047,0.06921599805355072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,4095,0.11845333377520244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,3,0.022783999641736347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1,0.02237333357334137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,8191,0.19274665911992392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,7,0.022357332209746044
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,15,0.02276266614596049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,31,0.022682666778564453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,63,0.022863999009132385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,127,0.022789334257443745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,255,0.02762666592995326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1023,0.06937066713968913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,511,0.048581331968307495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,2047,0.11425066987673442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,4095,0.19000534216562906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1,0.013189333180586496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,8191,0.34595731894175213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,3,0.01310933381319046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,7,0.013349333157142004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,15,0.013365333278973898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,31,0.01341333364446958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,63,0.013349333157142004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,15,0.6495199998219808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,127,0.013354666531085968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,511,0.024005333582560223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,255,0.020400000115235645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1023,0.02903466671705246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,8191,0.10755733648935954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,3,0.01993600030740102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,2047,0.04455466568470001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,7,0.019999999552965164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,15,0.020026666422684986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,4095,0.06523733337720235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,31,0.657807985941569
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,31,0.020021333048741024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1,0.02006400004029274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,63,0.020047999918460846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,127,0.02027200038234393
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,255,0.02754666656255722
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,511,0.03211733450492223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1023,0.04653333127498627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,4095,0.11290666460990906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1,0.022431999444961548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,8191,0.19364267587661743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,31,0.022613334159056347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,63,0.022848000129063923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,7,0.02269333352645238
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,127,0.0232640008131663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,15,0.022698665658632915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,2047,0.06940266489982605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,255,0.0276053324341774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,2047,0.11517332990964253
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,511,0.05072000126043955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,4095,0.1934559941291809
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,3,0.022639999787012737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1023,0.0713973343372345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1,0.026895999908447266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,3,0.02703999976317088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,8191,0.3492159843444824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,7,0.027295999228954315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,15,0.027317332724730175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,31,0.027424000203609467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,127,0.027930667002995808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,63,0.02759466568628947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1023,0.12296533584594727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,255,0.04491733511288961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,2047,0.23565866549809775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,511,0.07125333448251088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,3,0.012266666938861212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,4095,0.41894932587941486
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,7,0.012389333297808966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,15,0.012351999680201212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,63,0.012437333663304647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,127,0.012533333152532578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,511,0.018618666877349217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,255,0.01811733345190684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1023,0.019546666493018467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,2047,0.021322667598724365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,31,0.012730666746695837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,8191,0.8163626988728842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,8191,0.0328053335348765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,4095,0.025040000677108765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,3,0.01250133290886879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1,0.012527999778588613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,16383,0.04714666803677877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,32767,0.05228800078233083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,7,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,15,0.01250133290886879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,31,0.012655999511480331
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,63,0.012554666648308435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,127,0.01257066677014033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,511,0.018458666900793713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,255,0.018005333840847015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1023,0.01956266661485036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,2047,0.02141333371400833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,4095,0.025397333006064098
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,8191,0.03626133253177007
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,16383,0.04032533367474874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,32767,0.05823466678460439
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,3,0.012858666479587555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,7,0.012682666381200155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,15,0.012693333129088083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,31,0.012762666990359625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,63,0.012367999802033106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,127,0.012869333227475485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,255,0.01815466706951459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,511,0.018725333114465077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1023,0.01937066639463107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,2047,0.021477334201335907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,16383,0.04550399879614512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,8191,0.03346133232116699
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,4095,0.02845333268245061
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,32767,0.07227199772993724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1,0.012479999413092932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,7,0.012650666137536367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,3,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,15,0.01301866645614306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,31,0.012576000144084295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,63,0.012853333105643591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,127,0.012805332740147909
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,255,0.01825599993268649
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1023,0.02004266654451688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,511,0.01886933296918869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,2047,0.02478933334350586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,4095,0.029493334392706554
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,8191,0.040218666195869446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,16383,0.063509335120519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1,0.012373333175977072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,3,0.012282667060693106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,32767,0.10652800401051839
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,7,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,15,0.01268799975514412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,31,0.012730666746695837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,63,0.01250133290886879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,127,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,511,0.018751999984184902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,255,0.018346666047970455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1023,0.0200853335360686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,4095,0.02584533393383026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,2047,0.021925332645575207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,8191,0.03570666660865148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,16383,0.04050666590531667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,32767,0.0612960010766983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,3,0.0180479995906353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1,0.018266666680574417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,7,0.018464000274737675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,15,0.018330667167901993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,127,0.01823466643691063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,255,0.022122666239738464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,31,0.018330667167901993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,63,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,511,0.022458667556444805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1023,0.023525332411130268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,2047,0.02532266577084859
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,4095,0.03346666693687439
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,8191,0.037808001041412354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,16383,0.05611200133959452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1,0.018245333184798557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,32767,0.0788266658782959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,3,0.01817600056529045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,7,0.018272000054518383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,15,0.018378666291634243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,31,0.019061333189407986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,63,0.018432000031073887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,127,0.018533332894245785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,255,0.02203733225663503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,511,0.0229066660006841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1023,0.023999998966852825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,2047,0.0290133332212766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,4095,0.03347733368476232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,8191,0.046911999583244324
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,16383,0.06975999971230824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,32767,0.1111893355846405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1,0.018698666244745255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,3,0.01865600049495697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,7,0.018602666755517323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,15,0.018757333358128864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,31,0.018357332795858383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,63,0.01882133384545644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,127,0.018981333822011948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,255,0.022517333428064983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1023,0.027210667729377747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,511,0.022991999983787537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,4095,0.042677332957585655
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,8191,0.06565866867701213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,16383,0.10817600289980571
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,2047,0.03196800003449122
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,32767,0.18743467330932617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1,0.013450667262077332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,7,0.013877333452304205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,3,0.013493333011865616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,15,0.01404800017674764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,31,0.01370666672786077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,63,0.01381333296497663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,127,0.013999999811251959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,255,0.02478933334350586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1,0.02279466638962428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,511,0.029152000943819683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,2047,0.06549333532651265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1023,0.046351999044418335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,3,0.02309333284695943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,7,0.022944000860055287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,15,0.023061332603295643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,63,0.02348800003528595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,127,0.023786666492621105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,31,0.023018665611743927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,4095,0.10870400071144104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,255,0.028069332242012024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,511,0.05402133365472158
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,4095,0.19740800062815347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,3,0.02717333287000656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1023,0.07417599856853485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,7,0.027562665442625683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1,0.02739199995994568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,15,0.027797333896160126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,31,0.027600000301996868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,2047,0.12216533223787944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,63,0.027637332677841187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,127,0.028618666032950085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,255,0.047279998660087585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,2047,0.22425599892934164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,511,0.07351999978224437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1,0.03335466732581457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,3,0.033957332372665405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,7,0.034154665966828666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,15,0.03428266694148382
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,4095,0.4218133290608724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1023,0.12288533647855122
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,63,0.03448000053564707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,31,0.034389334420363106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,127,0.044026667873064675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,3,0.012885333349307379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1023,0.19268266359965006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,7,0.012821332861979803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,255,0.06816533207893372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,2047,0.3563786745071411
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,15,0.012975999464591345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,63,0.012565333396196365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,31,0.012746666868527731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,511,0.10992532968521118
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,255,0.01838933303952217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1023,0.020207999895016353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,511,0.01877333347996076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,2047,0.02250666668017705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,127,0.012629333883523941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,8191,0.03403733422358831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,16383,0.05013333261013031
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,32767,0.07473066449165344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,3,0.01826133330663045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1,0.01841066653529803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,7,0.01850133389234543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,15,0.01858666663368543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,63,0.018415999909241993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,31,0.018538666268189747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,4095,0.6816159884134928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,4095,0.029626667499542236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,127,0.018677332748969395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,255,0.022304000953833263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,511,0.023082666099071503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1023,0.02420799930890401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,4095,0.03369600077470144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,16383,0.07431999842325847
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,2047,0.029029332101345062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,8191,0.04923733572165171
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1,0.01869333287080129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,32767,0.11617066462834676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,3,0.018405333161354065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,7,0.018730666488409042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,15,0.018911999960740406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,255,0.02260799954334895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,127,0.018944000204404194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,63,0.018768000106016796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,31,0.018768000106016796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,511,0.0936906635761261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,2047,0.03197866678237915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,4095,0.04577599962552389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,8191,0.06798399984836578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,16383,0.10912533601125081
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1,0.01855466639002164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1023,0.027402666707833607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,32767,0.19325333833694458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,3,0.01883200059334437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,7,0.019061333189407986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,15,0.01918399954835574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,31,0.018906666586796444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,127,0.018794666975736618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,255,0.02275199939807256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,63,0.019093333433071773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1023,0.031130666534105938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,511,0.026394667724768322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,2047,0.042122667034467064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,4095,0.06542400022347768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,8191,0.10669333736101787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,16383,0.20061333974202475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,32767,0.3382026751836141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1,0.016224000602960587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,3,0.016501333564519882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,7,0.01632000009218852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,15,0.016602666427691776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,63,0.017008000363906223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,31,0.01642666632930438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,127,0.017349333812793095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,255,0.021781332790851593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1,0.02943466603755951
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,511,0.0555626650651296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,3,0.029696000119050343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1023,0.07670933504899342
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,7,0.029696000119050343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,15,0.029711998999118805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,31,0.029674666623274486
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,2047,0.12157332897186279
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,63,0.030069333811601002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,127,0.03071466585000356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,255,0.04929600159327189
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,511,0.07565333445866902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1,0.03625600039958954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1023,0.12763733665148416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,7,0.03682133307059606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,2047,0.22819199164708456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,15,0.036533333361148834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,3,0.03605866680542628
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,63,0.03745600084463755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,31,0.03682666768630346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,255,0.06935999790827434
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,2047,0.36090131600697833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1,0.05184000233809153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,3,0.05216533442338308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,127,0.04671466847260793
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,511,0.11261866490046184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,15,0.052709331115086876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,31,0.053717335065205894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,7,0.05276266733805338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,63,0.05860800047715505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,127,0.06809066732724507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1023,0.19705599546432495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,255,0.11077333490053813
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1023,0.34306132793426514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,2047,0.6450506846110026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,1,0.021530665457248688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,3,0.02130666623512904
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,511,0.20045334100723267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,31,0.02176533391078313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,63,0.021984001000722248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,15,0.021935999393463135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,7,0.02182399978240331
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,127,0.022805333137512207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,255,0.043178667624791466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,511,0.06909866631031036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,1,0.041002665956815086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,3,0.041120000183582306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,7,0.041290665666262306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,15,0.04137066751718521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,31,0.04178133110205332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,63,0.04330666859944662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,127,0.0536053329706192
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,1,0.05850133299827576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,255,0.07754133145014445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,511,0.12382933497428894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,3,0.0584746648867925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,7,0.0592853327592214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,15,0.059215997656186424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,31,0.06126933296521505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,63,0.06830933193365733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,127,0.07813866436481476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,255,0.12193600336710612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,1,0.08895466725031535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,511,0.20393067598342896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,3,0.08966933687527974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,7,0.09123200178146362
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,15,0.09587732950846355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,31,0.09768533706665039
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,63,0.09993066390355428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,127,0.11889599760373433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,255,0.20384534200032553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,511,0.35874664783477783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,1,0.02977066735426585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,3,0.030133334298928578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,7,0.030533333619435627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,15,0.030506665507952373
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,31,0.030975999931494396
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,63,0.036661334335803986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,1,0.06599466502666473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,127,0.046896000703175865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,3,0.06631466746330261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,255,0.0688213308652242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,7,0.06758399804433186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,15,0.06977599859237671
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,31,0.07474133372306824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,63,0.07850133379300435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,127,0.0918933351834615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,255,0.13895466923713684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,1,0.09921066959698994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,3,0.10016533732414246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,7,0.1053493320941925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,15,0.10843732953071594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,31,0.10893332958221436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,63,0.112527996301651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,127,0.13414399822553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,255,0.22137600183486938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,1,0.1600213348865509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,3,0.16799465815226236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1,0.012794667234023413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,3,0.01259200026591619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,7,0.1740106741587321
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,7,0.012821332861979803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,31,0.012778667112191519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,15,0.17280532916386923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,15,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,127,0.012778667112191519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,511,0.01916266605257988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,31,0.17193599541982016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,255,0.0185759998857975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1023,0.02085866779088974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,63,0.17389865716298422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,2047,0.02611200014750163
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,16383,0.07046933472156525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,4095,0.0305173322558403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1,0.018992000569899876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,8191,0.04808000226815542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,127,0.21360532442728677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,3,0.018725333114465077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,15,0.01924266666173935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,7,0.01887999971707662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,31,0.019120000302791595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,63,0.018965333700180054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,32767,0.11901332934697469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,255,0.02276266614596049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,511,0.023082666099071503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,4095,0.04845866560935974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1023,0.027877333263556164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,2047,0.032186667124430336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,8191,0.06754666566848755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,16383,0.11288000146547954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,7,0.01970133309563001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1,0.01960533360640208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,32767,0.20721065998077393
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,3,0.019199999670187633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,15,0.019472000499566395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,31,0.019600000232458115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,63,0.019733333339293797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,255,0.02295999974012375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,127,0.01932799940307935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,511,0.02641066660483678
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,2047,0.044480000933011375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,255,0.3750346501668294
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,4095,0.06822933256626129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1023,0.03134933362404505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,8191,0.10724799831708272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,16383,0.1918506622314453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1,0.019152000546455383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,15,0.019519999623298645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,32767,0.3429439862569173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,7,0.01960533360640208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,31,0.01930133377512296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,63,0.019440000255902607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,127,0.019760000209013622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,255,0.026250667870044708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1023,0.04284266630808512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,2047,0.06655466556549072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,511,0.030847998956839245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,127,0.019248000035683315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,4095,0.10764267047246297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,8191,0.18726933002471924
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,16383,0.34162131945292157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,32767,0.6513226826985677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,1,0.05211733281612396
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,3,0.05292266607284546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,7,0.057402665416399636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,15,0.058389330903689064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,31,0.06036800146102905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,63,0.065461332599322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,3,0.0194560003777345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,1,0.12597866853078207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,127,0.07960533102353413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,15,0.13105600078900656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,3,0.1288373370965322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,7,0.1306826670964559
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,63,0.13586666186650595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,31,0.13105066617329916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,1,0.19276267290115356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,3,0.19678932428359985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,7,0.19991467396418253
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,15,0.197818656762441
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,31,0.19855467478434244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,63,0.2036959926287333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,1,0.31623999277750653
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,127,0.2536533276240031
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,3,0.32179733117421466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,7,0.3255573312441508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,15,0.32385067145029706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,31,0.32196799914042157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,63,0.33052267630894977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,127,0.1646880010763804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,127,0.4322880109151204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,1,0.10372266173362732
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,7,0.10593066612879436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,3,0.10547199845314026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,15,0.1065120001633962
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,63,0.11132267117500305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,31,0.10498666763305664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,1,0.2398293415705363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,3,0.2406933307647705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,7,0.2443466583887736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,31,0.24279467264811197
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,15,0.24572799603144327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,1,0.3758293390274048
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,3,0.37504533926645917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,63,0.24913066625595093
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,7,0.3810186783472697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,15,0.3800693353017171
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,31,0.38311465581258136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,1,0.6175253391265869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1,0.013061333447694778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,3,0.013210666676362356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,7,0.013189333180586496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,15,0.013066666821638743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,63,0.3868853251139323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,31,0.013013333082199097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,63,0.013183999806642532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,3,0.6255360047022501
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,127,0.013280000537633896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,511,0.020314666132132213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1023,0.02493866781393687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,8191,0.06516266862551372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,2047,0.02938666691382726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,4095,0.0441599984963735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,255,0.019402666638294857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,16383,0.11050666371981303
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,7,0.019776000330845516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1,0.019706666469573975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,3,0.01977066695690155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,15,0.01979200045267741
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,7,0.6308853228886923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,31,0.019632000476121902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,127,0.019727999965349834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,63,0.01971199984351794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,255,0.02372266600529353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,511,0.027109332382678986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1023,0.03217600037654241
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,8191,0.11288533608118693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,16383,0.1910933256149292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,2047,0.044981335600217186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1,0.019861333072185516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,4095,0.07072000205516815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,3,0.020096000283956528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,7,0.020101333657900494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,31,0.01998399943113327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,15,0.020143999407688778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,63,0.01972266659140587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,127,0.020527999848127365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,255,0.026885333160559338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,511,0.031717332700888314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1023,0.04619200030962626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,2047,0.07220800220966339
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,4095,0.10945066809654236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,8191,0.19091200828552246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1,0.02239999920129776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,3,0.02239466706911723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,16383,0.348527987798055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,7,0.022672000030676525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,15,0.022602667411168415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,31,0.022645334402720135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,63,0.02279466638962428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,127,0.023232000569502514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,255,0.027999999622503918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,511,0.047482664386431374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1023,0.06944533189137776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,2047,0.11210133632024129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,4095,0.19100799163182577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,8191,0.34196801980336505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,15,0.6295946836471558
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1,0.01310933381319046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,3,0.012975999464591345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,7,0.013429333766301474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,16383,0.6467039982477824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,15,0.013306666165590286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,31,0.013354666531085968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,127,0.013493333011865616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,255,0.01993600030740102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,511,0.023333333432674408
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,31,0.6378293434778849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1023,0.028245332340399425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,8191,0.11158399780591328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,2047,0.04278933505217234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1,0.019866666446129482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,7,0.019968000551064808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,63,0.013455999394257864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,4095,0.06437333424886067
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,31,0.01985599969824155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,15,0.020224000016848247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,63,0.020074666788180668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,127,0.02027733375628789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,511,0.031727999448776245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1023,0.04520533482233683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,255,0.02720533311367035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,4095,0.11297067006429036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,3,0.019968000551064808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,2047,0.06993066767851512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,7,0.022917332748572033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,63,0.6408106486002604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,3,0.02250666668017705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,8191,0.1922666629155477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1,0.022602667411168415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,15,0.023029332359631855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,63,0.023157333334287006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,127,0.023503998915354412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,255,0.027877333263556164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,31,0.02256533255179723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,511,0.04957333207130432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,4095,0.19221333662668863
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,3,0.0268053337931633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1,0.02666666607062022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1023,0.07028799752394359
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,8191,0.3500479857126872
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,7,0.027269333600997925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,15,0.02739199995994568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,31,0.026906666656335194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,63,0.027130665878454845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,255,0.04230933388074239
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,2047,0.22262932856877646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,4095,0.41810667514801025
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,127,0.028143999477227528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1023,0.12846400340398154
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,3,0.012389333297808966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,511,0.07163733243942261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,8191,0.8190346558888754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1,0.012240000069141388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,7,0.01240533341964086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,15,0.012469333906968435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,31,0.01250133290886879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,511,0.01848000039656957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,255,0.018079999834299088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1023,0.019039999693632126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,2047,0.021359999974568684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,8191,0.03149333347876867
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,4095,0.024346667031447094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,16383,0.04714666803677877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1,0.012362666428089142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,32767,0.05282133320967356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,127,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,7,0.012485332787036896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,3,0.012506666282812754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,15,0.01267733300725619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,31,0.012527999778588613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,127,0.012506666282812754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,63,0.012330666184425354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,255,0.017994667092959087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,511,0.01830400029818217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,2047,0.021087999145189922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1023,0.019343999524911244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,4095,0.024512000381946564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,8191,0.03525333354870478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1,0.01240533341964086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,16383,0.03925333420435587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,32767,0.05459733307361603
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,3,0.012485332787036896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,7,0.012831999609867731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,15,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,31,0.012634667257467905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,63,0.012458667159080505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,127,0.012597333639860153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,255,0.018138666947682697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,2047,0.021802666286627453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1023,0.019744000087181728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,511,0.01846933364868164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,4095,0.02809600035349528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,8191,0.03267733256022135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,16383,0.04464533428351084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1,0.012479999413092932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,2047,0.11361066500345866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,3,0.012736000120639801
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,7,0.01259200026591619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,32767,0.07141333321730296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,15,0.012821332861979803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,31,0.012597333639860153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,63,0.012629333883523941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,127,0.012768000364303589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1023,0.019930666933457058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,511,0.018698666244745255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,255,0.018218666315078735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,2047,0.025050667424996693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,4095,0.029477333029111225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,8191,0.040864000717798867
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,16383,0.06320533156394958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,32767,0.10499733686447144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,3,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,7,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,127,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,15,0.012618667135636011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,31,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,255,0.01817600056529045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,511,0.018394666413466137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,2047,0.021509334444999695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1023,0.01959466685851415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,4095,0.025450666745503742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,16383,0.040048000713189445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,8191,0.034661332766215004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1,0.018373332917690277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,32767,0.059861332178115845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,3,0.018432000031073887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,7,0.0182239996890227
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,31,0.018239999810854595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,127,0.018533332894245785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,15,0.01825599993268649
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,63,0.01791999985774358
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,255,0.02181333303451538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,511,0.02248000105222066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1023,0.023226665953795116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,4095,0.03197866678237915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,8191,0.036714665591716766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,2047,0.025221332907676697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,16383,0.05523733297983805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1,0.018453333526849747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,32767,0.07937600215276082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,7,0.018640000373125076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,3,0.01855466639002164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,15,0.018458666900793713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,127,0.018618666877349217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,31,0.018320000420014065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,63,0.018309333672126133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,511,0.022367998957633972
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,255,0.02189333240191142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1023,0.023754666248957317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,2047,0.028768000503381092
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,4095,0.03333866596221924
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,8191,0.046480000019073486
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,16383,0.06956266860167186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,32767,0.11200533310572307
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,3,0.01836799954374631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,15,0.01884799947341283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,7,0.018581333259741466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,31,0.01863466699918111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,63,0.01842133328318596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,255,0.02203733225663503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,127,0.01887999971707662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,511,0.02271466702222824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1023,0.0271519993742307
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,2047,0.031914666295051575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,4095,0.04252799848715464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,8191,0.06540800134340923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,16383,0.10877866546312968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,32767,0.18654932578404745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1,0.01340266689658165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,3,0.013557333499193192
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,7,0.013568000247081121
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,15,0.013642666240533194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,31,0.013605333864688873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,63,0.013493333011865616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,127,0.013829333086808523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,255,0.023951999843120575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,511,0.028757333755493164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1,0.022485333184401195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,3,0.022592000663280487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,7,0.02274666726589203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1023,0.04539733131726583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,15,0.022917332748572033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,31,0.022821334501107533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,63,0.023141334454218548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,2047,0.06449600060780843
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,127,0.023658665517965954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,255,0.027973333994547527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,511,0.05269333223501841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,4095,0.10914666453997295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,4095,0.19650665918986002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1023,0.0727040022611618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,7,0.02757866680622101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,3,0.02740799884001414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1,0.026911998788515728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,15,0.027530667682488758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,2047,0.11901866396268208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,63,0.02752533306678136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,255,0.04599999884764353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,2047,0.22476800282796225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,127,0.028533334533373516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,511,0.07315200070540111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1,0.03329599897066752
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,31,0.027562665442625683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1023,0.12309867143630981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,7,0.03421333432197571
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,3,0.033904001116752625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,4095,0.4195733467737834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,31,0.03401600072781245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,63,0.03443733354409536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1023,0.19252800941467285
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,15,0.03428266694148382
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,255,0.06658666829268138
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,127,0.04366933306058248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,3,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1,0.01249066616098086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,2047,0.3561013142267863
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,7,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,15,0.01267733300725619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,511,0.11000000437100728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,63,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,127,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,255,0.018325333793958027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,511,0.01903466631968816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1023,0.01987733319401741
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,4095,0.6842986742655436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,8191,0.03379733363787333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,16383,0.04832000037034353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,4095,0.028880000114440918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1,0.01814933369557063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,32767,0.0717439999183019
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,3,0.018559999763965607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,2047,0.022170667846997578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,7,0.01838933303952217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,15,0.018543999642133713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,63,0.01838933303952217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,31,0.018522666146357853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,127,0.0186666672428449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,255,0.02207999924818675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,511,0.022778667509555817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,4095,0.033600000043710075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,2047,0.029045333464940388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1023,0.02386666586001714
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,16383,0.07133333384990692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,8191,0.05014933149019877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1,0.018746666610240936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,3,0.018511999398469925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,32767,0.12216533223787944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,7,0.01889066646496455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,15,0.018906666586796444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,31,0.01899733394384384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,63,0.01886933296918869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,127,0.018976000448067982
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,255,0.022341333329677582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,2047,0.03193599979082743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1023,0.02739733209212621
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,4095,0.04389866689840952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,511,0.02309866746266683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,8191,0.06712000072002411
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,16383,0.1086186667283376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,32767,0.19300800561904907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1,0.018672000616788864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,7,0.01897066707412402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,3,0.018757333358128864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,15,0.018794666975736618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,63,0.018954666952292126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,127,0.018768000106016796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,31,0.019226666539907455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,255,0.022650666534900665
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,511,0.026149332523345947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1023,0.031136001149813335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,2047,0.04229333500067393
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,4095,0.06548800071080525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,8191,0.10588266452153523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,16383,0.1855306625366211
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,32767,0.36988266309102374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1,0.01634666696190834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,3,0.016469333320856094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,7,0.01664000004529953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,15,0.01672533278663953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,31,0.016565332810084026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,63,0.016976000120242436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,127,0.017386666188637417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,255,0.02203733225663503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,511,0.05082666873931885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1,0.02804800122976303
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,3,0.02804800122976303
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,7,0.028570666909217834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1023,0.06993066767851512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,15,0.028357334434986115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,31,0.028751999139785767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,63,0.02845866729815801
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,127,0.02938666691382726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,2047,0.1167146662871043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,255,0.047024001677831016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1,0.03499199946721395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,511,0.07492800056934357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1023,0.12712533275286356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,3,0.03482666611671448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,7,0.03550933301448822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,2047,0.22748800118764242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,31,0.035232000052928925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,127,0.04487466812133789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,15,0.035674666364987694
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,255,0.06880533198515575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1023,0.19478933016459146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,511,0.11316800117492676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,2047,0.3585386673609416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1,0.05035733183224996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,3,0.05089599887530009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,7,0.0513919989267985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,127,0.06685866912206014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,31,0.05233600238958994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,15,0.051872000098228455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,255,0.10756799578666687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,3,0.02103466788927714
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,2047,0.6423519849777222
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1023,0.34172264734903973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,7,0.02146133283774058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,511,0.20052266120910645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,1,0.02107733239730199
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,15,0.021386665602525074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,31,0.021216000119845074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,255,0.04205333193143209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,511,0.06834666430950165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,127,0.02293333411216736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,1,0.0383146678407987
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,3,0.03822399924198786
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,7,0.03881600002447764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,15,0.03874133278926214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,31,0.03880000114440918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,63,0.03995733211437861
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,127,0.04980800052483877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,255,0.07344000041484833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,1,0.05584533512592316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,3,0.056186666091283165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,511,0.12166933218638103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,7,0.05653866628805796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,63,0.06437866886456807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,31,0.05774933099746704
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,15,0.057248001297314964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,127,0.07595733304818471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,255,0.11708799997965495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,1,0.08630399902661641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,3,0.08678399523099263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,7,0.09014933307965596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,511,0.19649600982666016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,15,0.09103467067082723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,31,0.0942133367061615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,63,0.09577600161234538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,255,0.1983840068181356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,127,0.1120799978574117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,511,0.3503520091374715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,1,0.028768000503381092
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,3,0.028917332490285236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,7,0.02932800104220708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,15,0.02922133356332779
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,63,0.03136000037193298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,127,0.04306666553020477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,1,0.06197333335876465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,31,0.029317334294319153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,255,0.06666133304437001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,31,0.067930668592453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,15,0.06305066744486491
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,3,0.061834668119748436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,7,0.06277866661548615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,63,0.07307200133800507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,1,0.09438932935396831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,127,0.08499200145403545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,3,0.09653866291046143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,7,0.09868266185124715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,255,0.1309279998143514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,31,0.10391466816266377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,15,0.10294933120409648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,63,0.10750400026639302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,127,0.1279146671295166
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,255,0.2128159999847412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,1,0.1549493372440338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,3,0.15787200133005777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,7,0.16703999042510986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,15,0.1691840092341105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1,0.012613333761692047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,31,0.1670080025990804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,15,0.012789333860079447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,7,0.012709333250919977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,31,0.012847999731699625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,63,0.012725333372751871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,127,0.012821332861979803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,255,0.018624000251293182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,511,0.019226666539907455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1023,0.020400000115235645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,2047,0.025653332471847534
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,4095,0.030085332691669464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,63,0.16884267330169678
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,8191,0.044954667488733925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,16383,0.0664106657107671
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,32767,0.11116266250610352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,3,0.0191040001809597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1,0.018719999740521114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,7,0.019194666296243668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,15,0.019189332922299702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,127,0.20739734172821045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,31,0.019343999524911244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,63,0.01905599981546402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,127,0.019189332922299702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,511,0.022944000860055287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1023,0.02743999908367793
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,2047,0.031925333042939506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,4095,0.04645866652329763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,255,0.022474666436513264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,255,0.36851731936136883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,8191,0.06776533524195354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,16383,0.11359467109044392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,32767,0.20579200983047485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1,0.019002666076024372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,7,0.019578666736682255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,31,0.01929066702723503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,15,0.01924266666173935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,63,0.019546666493018467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,127,0.0191040001809597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,255,0.022831998765468597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,511,0.026533332963784535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1023,0.030991998811562855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,2047,0.04413333535194397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,4095,0.06724800169467926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,8191,0.10806399583816528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,16383,0.18865066766738892
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,3,0.019189332922299702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1,0.018986667195955913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,32767,0.34325865904490155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,3,0.019066666563351948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,7,0.019632000476121902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,31,0.01916266605257988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,63,0.019695999721686046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,15,0.019626667102177937
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,127,0.01945066700379054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,255,0.026362667481104534
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1023,0.04200000067551931
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,511,0.030591999491055805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,2047,0.06860266625881195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,4095,0.10852266351381938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,8191,0.18611733118693033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,16383,0.3394453525543213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,32767,0.6478933493296305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,1,0.04844800134499868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,3,0.049327999353408813
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,7,0.05016533533732096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,15,0.05384533107280731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,31,0.05712000032265981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,1,0.11141332983970642
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,3,0.11652800440788269
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,63,0.060133333007494606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,7,0.11946666240692139
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,127,0.07391466697057088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,15,0.12062933047612508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,31,0.12145066261291504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,63,0.12499200304349263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,1,0.17939732472101846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,127,0.1525173286596934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,3,0.18531199296315512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,7,0.19007466236750284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,15,0.18838399648666382
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,63,0.19316800435384116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,31,0.18769067525863647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,3,0.3117226759592692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,127,0.2362133264541626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,7,0.31571733951568604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,15,0.31381867329279584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,1,0.30345600843429565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,63,0.31973334153493244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,31,0.3133546710014343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,127,0.40019198258717853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,1,0.09798933068911235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,3,0.09875200192133586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,15,0.10046399633089702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,7,0.10055466492970784
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,31,0.09940800070762634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,63,0.10358933607737224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,1,0.2183306614557902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,3,0.21973333756128946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,7,0.22337599595387778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,15,0.22267200549443564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,31,0.2214453419049581
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,63,0.2273706595102946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,1,0.3538186550140381
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,3,0.35598933696746826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,7,0.35948801040649414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,15,0.35913066069285077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,31,0.3625013430913289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,63,0.3661653200785319
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,1,0.5970240036646525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,3,0.6019733349482218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1,0.01312000056107839
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,7,0.6102133194605509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,3,0.013002666334311167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,7,0.013082666943470636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,15,0.6118880112965902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,15,0.01321600005030632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,31,0.013066666821638743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,127,0.013125333935022354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,63,0.013114667187134424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,255,0.0191040001809597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,511,0.01974933346112569
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1023,0.024197332561016083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,2047,0.02886933336655299
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,4095,0.04308266441027323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,8191,0.06420266628265381
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,16383,0.10903466741243999
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1,0.019498666127522785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,3,0.020341333001852036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,7,0.019674666225910187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,15,0.01978133370478948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,31,0.01970133309563001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,63,0.019648000597953796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,255,0.023317334552605946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,31,0.6143680016199747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,127,0.01958400011062622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,511,0.026799999177455902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1023,0.03172266731659571
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,8191,0.11110400160153706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,2047,0.044581333796183266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,16383,0.1890880068143209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,63,0.6196000178654989
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1,0.019882666567961376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,7,0.020015999674797058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,31,0.020202666521072388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,63,0.019893333315849304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,3,0.019578666736682255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,127,0.02046400060256322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,255,0.02666666607062022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,511,0.030933332939942677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,2047,0.06894933183987935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1023,0.04322666426499685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,8191,0.19265600045522055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,4095,0.1090719997882843
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1,0.022133332987626392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,7,0.022469334304332733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,3,0.022128000855445862
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,16383,0.34781865278879803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,15,0.022442666192849476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,31,0.022133332987626392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,63,0.022821334501107533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,127,0.023061332603295643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,4095,0.06996266543865204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,255,0.027386667827765148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1023,0.06751466790835063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,2047,0.11240532994270325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,511,0.04754666487375895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,8191,0.341813325881958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,16383,0.6439306735992432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,4095,0.1872533361117045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,3,0.01321600005030632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,7,0.013151999562978745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,15,0.013221333424250284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,31,0.013157332936922709
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,63,0.013151999562978745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,127,0.013173333058754602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1,0.012949333836634954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,255,0.01947733387351036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,511,0.02349333216746648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1023,0.027749332288901012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,8191,0.10789333780606587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,2047,0.04135466615358988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,4095,0.06414400041103363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1,0.019578666736682255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,16383,0.18875734011332193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,15,0.019909333437681198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,31,0.019808000574509304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,3,0.019637333850065868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,63,0.019999999552965164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,127,0.020058666666348774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,7,0.01998399943113327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,255,0.026709333062171936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,15,0.0199946661790212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,511,0.031173333525657654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1023,0.043680002291997276
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,4095,0.11239999532699585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,2047,0.067930668592453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1,0.02258666604757309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,3,0.022656001150608063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,15,0.02260799954334895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,7,0.02258133391539256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,63,0.02313599983851115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,31,0.02269333352645238
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,127,0.023472001155217487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,255,0.027717334528764088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,2047,0.1113920013109843
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,511,0.048453330993652344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,4095,0.18969599405924478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1023,0.0690826674302419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,8191,0.3481653531392415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1,0.026496000587940216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,3,0.02701866626739502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,16383,0.657098650932312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,15,0.02712533374627431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,31,0.027098665634791057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,7,0.0271519993742307
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,63,0.02731200059254964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,127,0.028005334238211315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,255,0.04135466615358988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1023,0.12177600463231404
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,511,0.07191466788450877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,2047,0.23494933048884073
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,4095,0.41950400670369464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1,0.012133333832025528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,3,0.01211200033624967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,7,0.012304000556468964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,8191,0.8131893475850424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,15,0.012517333030700684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,31,0.01232533281048139
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,63,0.012223999947309494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,127,0.012746666868527731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,255,0.017802666872739792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,511,0.018346666047970455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1023,0.01921066641807556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,2047,0.020799999435742695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,4095,0.024277334411938984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,8191,0.0305173322558403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,16383,0.04619733492533366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,16383,1.7214560508728027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,32767,0.052149335543314614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,65535,0.0670826683441798
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1,0.012304000556468964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,3,0.012560000022252401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,7,0.012986666212479273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,15,0.01251199965675672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,31,0.012565333396196365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,63,0.01259200026591619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,127,0.012565333396196365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,511,0.018432000031073887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,255,0.0179626668492953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1023,0.01922133316596349
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,2047,0.020831999679406483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,8191,0.03408000121514002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,4095,0.02465066562096278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,16383,0.03882666677236557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,32767,0.05462400118509928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1,0.012330666184425354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,3,0.012639999389648438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,65535,0.0827466646830241
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,7,0.012778667112191519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,15,0.012746666868527731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,31,0.012719999998807907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,63,0.01267733300725619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1023,0.01956266661485036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,255,0.018394666413466137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,2047,0.021290667355060577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,511,0.01859733338157336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,127,0.012560000022252401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,32767,0.06955199937025706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,4095,0.027855999767780304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,16383,0.04404800136884054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,8191,0.03270933280388514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,65535,0.11708266536394756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,3,0.012576000144084295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1,0.012576000144084295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,7,0.012762666990359625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,15,0.012858666479587555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,63,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,31,0.01268799975514412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,127,0.012842666357755661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,255,0.01833600054184596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,511,0.01884799947341283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1023,0.019845332950353622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,8191,0.040063999593257904
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,4095,0.02932800104220708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,16383,0.06297066807746887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,2047,0.02447466552257538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,32767,0.10503466924031575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1,0.012373333175977072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,3,0.01227733368674914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,65535,0.18743467330932617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,7,0.012618667135636011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,15,0.01259200026591619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,31,0.012506666282812754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,255,0.018058666338523228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,63,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,511,0.01854933301607768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,127,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1023,0.019573333362738293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,2047,0.02139200021823247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,8191,0.03527999917666117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,16383,0.041152000427246094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,4095,0.02552533398071925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,32767,0.05835733314355215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,65535,0.08291199803352356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1,0.018330667167901993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,3,0.018373332917690277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,7,0.018538666268189747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,63,0.01836799954374631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,31,0.018320000420014065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,15,0.018543999642133713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,127,0.018112000077962875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,255,0.021898667017618816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,8191,0.03710933278004328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,511,0.022448000808556873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,2047,0.025066666305065155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1023,0.023189333577950794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,4095,0.03196266790231069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,16383,0.05062933266162872
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1,0.018351999421914417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,65535,0.11998933553695679
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,32767,0.07805866499741872
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,3,0.018474667022625606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,7,0.018559999763965607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,15,0.018805333723624546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,31,0.01848000039656957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,63,0.01870399961868922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,127,0.018538666268189747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,255,0.022069332500298817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,511,0.02233600119749705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1023,0.023738667368888855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,2047,0.02846933404604594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,4095,0.03324266771475474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,8191,0.04468800127506256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,16383,0.06864533325036366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,32767,0.11150933305422465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1,0.01810666670401891
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,65535,0.19279466072718301
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,3,0.018325333793958027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,7,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,15,0.018378666291634243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,31,0.01854933301607768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,63,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,127,0.018581333259741466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,255,0.02197333425283432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,511,0.023007998863856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1023,0.02720533311367035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,2047,0.03156800071398417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,4095,0.042949333786964417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,8191,0.06570133566856384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,16383,0.10815466443697612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,32767,0.18709866205851236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,65535,0.33885331948598224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1,0.013295999417702356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,3,0.013349333157142004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,7,0.01341333364446958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,15,0.013471999516089758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,31,0.013397333522637686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,63,0.013349333157142004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,127,0.013744000345468521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,255,0.022991999983787537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,511,0.027722666660944622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1023,0.04094400008519491
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,2047,0.06451733410358429
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,8191,0.18397865692774454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1,0.022384000321229298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,3,0.022405333817005157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,4095,0.10698666175206502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,15,0.022511998812357586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,7,0.022656001150608063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,63,0.022848000129063923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,31,0.02235200007756551
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,255,0.02777066578467687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,511,0.050474668542544045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,127,0.023029332359631855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1023,0.07190399865309398
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,4095,0.1946720083554586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,16383,0.3752799828847249
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,2047,0.11820266644159953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1,0.026650667190551758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,8191,0.20675732692082724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,8191,0.35522135098775226
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,3,0.027104000250498455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,7,0.027210667729377747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,63,0.0272533322374026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,255,0.0443146675825119
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,15,0.02720533311367035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,2047,0.22444266080856323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,511,0.07212266822655995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,127,0.02845866729815801
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,31,0.027056001126766205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,4095,0.419866681098938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1,0.03330666571855545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1023,0.12161067128181458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,7,0.0341333324710528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,15,0.034202667574087776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,3,0.033743999898433685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,8191,0.8692959944407145
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,31,0.03416533271471659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,63,0.03414933383464813
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,127,0.0428959975639979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1023,0.19339734315872192
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,2047,0.3800106843312581
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1,0.012400000045696894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,3,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,255,0.06693333387374878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,4095,0.681493361790975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,15,0.012714666624863943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,31,0.01259200026591619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,7,0.012437333663304647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,511,0.11028266946474712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,127,0.012613333761692047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,255,0.018320000420014065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1023,0.019871999820073444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,63,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,511,0.01874133323629697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,8191,0.033701332906881966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,4095,0.029066666960716248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,8191,1.4435893694559734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,16383,0.04933333396911621
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,32767,0.07308800021807353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,2047,0.021850667893886566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,65535,0.114138662815094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1,0.018533332894245785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,3,0.01829333355029424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,7,0.018506667266289394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,31,0.01871466636657715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,15,0.01863466699918111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,63,0.0186666672428449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,127,0.01842133328318596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1023,0.023792001108328503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,255,0.02197333425283432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,511,0.022597332795461018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,4095,0.03382933388153712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,2047,0.028730665644009907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,8191,0.050554667909940086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,16383,0.07063466807206471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1,0.018730666488409042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,3,0.018746666610240936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,32767,0.12087999780972798
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,65535,0.20908266305923462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,7,0.018661333868900936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,15,0.01889066646496455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,31,0.018794666975736618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,63,0.01887999971707662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,127,0.018757333358128864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,255,0.022282667458057404
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,511,0.023002666731675465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1023,0.02697066714366277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,2047,0.031930667658646904
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,4095,0.044922664761543274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,8191,0.06931200126806895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,16383,0.10816533366839091
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1,0.018522666146357853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,3,0.018624000251293182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,32767,0.19144533077875772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,65535,0.34616533915201825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,7,0.018954666952292126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,31,0.018640000373125076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,63,0.01868266612291336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,15,0.018581333259741466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,127,0.018810667097568512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,255,0.022645334402720135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1023,0.030949334303538006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,2047,0.04178133110205332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,4095,0.06883733471234639
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,511,0.026047999660174053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,8191,0.10602133472760518
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,16383,0.18330132961273193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,32767,0.3394026756286621
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,65535,0.6467253367106119
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,3,0.016037333756685257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1,0.016042667130629223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,7,0.016442666451136272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,15,0.016399999459584553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,31,0.016458666572968166
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,63,0.016693333784739178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,127,0.016938666502634685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,255,0.021568000316619873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,511,0.04858666658401489
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,3,0.02722666660944621
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1023,0.06717333197593689
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1,0.027552001178264618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,7,0.02759466568628947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,15,0.027653334041436512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,2047,0.11365866661071777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,31,0.027589333554108936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,63,0.027893332143624622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,127,0.028917332490285236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,255,0.04637866715590159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,4095,0.19131733973821005
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1023,0.12525332967440286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,511,0.07302933434645335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,4095,0.42688000202178955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,3,0.0341333324710528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1,0.03425599883000056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,2047,0.22632000843683878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,31,0.034559999903043113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,15,0.03473600000143051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,63,0.03489066660404205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,127,0.04347200194994608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,7,0.034688000877698265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,255,0.06729066868623097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,511,0.11085866888364156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,2047,0.35655999183654785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1,0.04970666766166687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,3,0.05021866659323374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,4095,0.739349365234375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1023,0.19417067368825278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,7,0.05077866713205973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,31,0.05146133402983347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,15,0.050437331199645996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,63,0.0563679983218511
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,127,0.06783466537793477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,511,0.19862399498621622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1023,0.3380533456802368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,255,0.11451199650764465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,2047,0.6400746504465739
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,4095,1.3604159355163574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1,0.02037866661945979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,3,0.02063999945918719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,7,0.021066665649414062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,15,0.020938667158285778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,31,0.021141332884629566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,127,0.0215786670645078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,63,0.020970667401949566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,255,0.03834133346875509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1,0.03631466627120972
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,511,0.06720533470312755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1023,0.11642133196194966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,3,0.03605333218971888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,7,0.036501333117485046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,15,0.03692800054947535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,31,0.03678400069475174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,63,0.03754133234421412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,255,0.07070399820804596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1,0.053802669048309326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,127,0.04696533580621084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,511,0.11627733707427979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,3,0.05425600210825602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1023,0.20715200901031494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,7,0.05461333195368449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,15,0.05464000006516775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,31,0.05515733361244202
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,63,0.06169599791367849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,127,0.07037333150704701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,255,0.11370666821797688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1,0.08434133728345235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,511,0.20544532934824625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,3,0.08474666873613994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,7,0.08637866377830505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,15,0.08847999572753906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1023,0.34780267874399823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,63,0.09291733304659526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,31,0.09159466624259949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1023,0.6487146615982056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,127,0.1088213324546814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,255,0.19321600596110025
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,7,0.028229333460330963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,15,0.02826666583617528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,3,0.02775999903678894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,127,0.041402667760849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,31,0.028399998943010967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,511,0.3596800168355306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,255,0.06331733365853627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,63,0.028965334097544353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,1,0.027744000156720478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,511,0.1070186694463094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,1,0.05807999769846598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,3,0.05846933523813883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,15,0.05899733304977417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,7,0.05860800047715505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,63,0.0680159976085027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,255,0.12449066837628682
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,127,0.08025066554546356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,1,0.09156266848246257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,511,0.20229866107304892
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,3,0.09158933162689209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,7,0.09344533085823059
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,15,0.09672000010808308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,63,0.10213333368301392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,31,0.09974400202433269
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,127,0.12247467041015625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,255,0.20594133933385214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,1,0.1509760022163391
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,3,0.15313067038853964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,511,0.3601653178532918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,15,0.16474666198094687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,7,0.16006400187810263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1,0.012479999413092932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,31,0.16403733690579733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,7,0.012608000387748083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,3,0.012671999633312225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,31,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,63,0.01267733300725619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,127,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,15,0.012736000120639801
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,255,0.018437333405017853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,511,0.018895999838908512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,63,0.1648426651954651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1023,0.020293333878119785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,2047,0.025370667378107708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,4095,0.030053332448005676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,8191,0.046341334780057274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,32767,0.11090133587519328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1,0.018645333747069042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,65535,0.19227200746536255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,3,0.01854933301607768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,7,0.018789333601792652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,15,0.01833600054184596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,31,0.018496000518401463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,63,0.01869333287080129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,16383,0.06901333232720692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,127,0.018330667167901993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,255,0.022490667800108593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,127,0.19990932941436768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,511,0.02317333221435547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,2047,0.03164266546567281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1023,0.02739733209212621
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,8191,0.06791466474533081
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,16383,0.11264533797899882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,32767,0.19429866472880045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1,0.018496000518401463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,65535,0.35210665067036945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,3,0.019061333189407986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,4095,0.04930666585763296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,7,0.018709332992633183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,15,0.019226666539907455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,31,0.01903466631968816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,63,0.019018666197856266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,127,0.01877333347996076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,255,0.02266666789849599
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,511,0.026394667724768322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,2047,0.04224533339341482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,4095,0.06743999818960826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,8191,0.10670933127403259
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1023,0.030693332354227703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,16383,0.1897546648979187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,32767,0.3434720039367676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,255,0.358517328898112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1,0.01883200059334437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,3,0.018672000616788864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,7,0.019391999890406925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,15,0.019039999693632126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,65535,0.6550613244374593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,63,0.018800000349680584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,511,0.6646773417790731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,127,0.019461333751678467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,255,0.02589333305756251
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1023,0.04114133367935816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,511,0.03054933249950409
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,4095,0.10691199700037639
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,8191,0.1869759956995646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,16383,0.34025601545969647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,32767,0.6471466620763143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,65535,1.3798240025838215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,31,0.01932799940307935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,31,0.061535999178886414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,2047,0.06524799764156342
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,1,0.04675200084845225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,3,0.046725332736968994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,7,0.0476800004641215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,15,0.04867733518282572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,31,0.05109866460164388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,127,0.07002133131027222
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,63,0.05788266658782959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,255,0.1162559986114502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,1,0.09794666369756062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,3,0.0995199978351593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,7,0.105813334385554
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,15,0.10794132947921753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,31,0.11143466830253601
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,63,0.11469333370526631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,127,0.14386133352915445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,255,0.220960001150767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,1,0.16383999586105347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,3,0.1699626644452413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,7,0.17845332622528076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,15,0.17795199155807495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,31,0.1779573361078898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,63,0.18523732821146646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,127,0.23328532775243124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,255,0.3816053469975789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,1,0.28706133365631104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,3,0.2991146643956502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,7,0.3073493242263794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,15,0.30541332562764484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,31,0.3031093279520671
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,63,0.3131360014279683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,127,0.4073226849238078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,255,0.693173329035441
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,1,0.090421328941981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,3,0.09259733557701111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,7,0.09516266981760661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,15,0.09618666768074036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,63,0.09760533769925435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,31,0.09476799766222636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,127,0.12762666742006937
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,1,0.19359999895095825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,3,0.19779733816782633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,7,0.20150933663050333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,15,0.2016106645266215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,31,0.20163732767105103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,63,0.20494933923085532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,1,0.3312106728553772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,3,0.3349066575368245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,127,0.25596799453099567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,7,0.3392000198364258
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,63,0.34466131528218585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,15,0.3373759984970093
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,31,0.34029332796732586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,1,0.5744586785634359
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1,0.012906666845083237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,3,0.012965332716703415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,127,0.4273546536763509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,7,0.012954667210578918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,3,0.5831626653671265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,15,0.012986666212479273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,7,0.5931359926859537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,63,0.013023999830087027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,31,0.01312000056107839
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,255,0.01874133323629697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,511,0.019551999866962433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1023,0.023775999744733173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,127,0.013157332936922709
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,2047,0.028666667640209198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,4095,0.04125866790612539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,8191,0.06587733328342438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,16383,0.10859200358390808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1,0.019317333896954853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,32767,0.18772266308466592
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,3,0.019317333896954853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,7,0.019343999524911244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,15,0.019493332753578823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,31,0.019589333484570186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,63,0.019359999646743137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,127,0.019637333850065868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,255,0.022789334257443745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,511,0.026533332963784535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1023,0.030767999589443207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,4095,0.06645333270231883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,2047,0.04483733574549357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,8191,0.11068800091743469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,16383,0.19024000565210977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1,0.019637333850065868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,15,0.5885440111160278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,3,0.019551999866962433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,7,0.019861333072185516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,32767,0.3763146797815959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,63,0.019930666933457058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,31,0.019626667102177937
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,127,0.019861333072185516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,255,0.026176000634829204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,511,0.03073599934577942
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,15,0.019930666933457058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,2047,0.06853866577148438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,4095,0.11003200213114421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1023,0.04387733340263367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,31,0.5955893198649088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,8191,0.2041920026143392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,16383,0.34786665439605713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1,0.021920000513394673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,7,0.02237333357334137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,3,0.022592000663280487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,15,0.022522665560245514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,32767,0.6584693193435669
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,63,0.022831998765468597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,127,0.022650666534900665
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1023,0.06688533226648967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,255,0.027610667049884796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,511,0.04669866462548574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,2047,0.1111253301302592
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,4095,0.18801599740982056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,8191,0.3410666783650716
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,31,0.02250666668017705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,16383,0.6477546691894531
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,3,0.012975999464591345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,7,0.013280000537633896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,15,0.013349333157142004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,31,0.013066666821638743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,63,0.013221333424250284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,127,0.013157332936922709
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,255,0.01951466624935468
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,511,0.022869333624839783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,32767,1.3765494028727214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1023,0.02757866680622101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,8191,0.11201066772143047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1,0.019509332875410717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,16383,0.18498132626215616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,2047,0.04260266820589701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,3,0.019744000087181728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,7,0.019839999576409657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,15,0.019914666811625164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,31,0.019930666933457058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,4095,0.062309334675470986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,63,0.019834666202465694
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,127,0.02013333390156428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,255,0.026501332720120747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,63,0.5959626833597819
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,2047,0.06742933392524719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1023,0.043375998735427856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,511,0.031023999055226643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,4095,0.11415466666221619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,8191,0.19036267201105753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,16383,0.37174399693806964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1,0.02235200007756551
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,31,0.02231466770172119
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,3,0.022458667556444805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,63,0.022869333624839783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,15,0.022309333086013794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,127,0.023381332556406658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,255,0.027530667682488758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,2047,0.1129866639773051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,4095,0.19022933642069498
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,511,0.04826666911443075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1,0.012298667182525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1023,0.069733331600825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,3,0.0124746672809124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,7,0.012448000411192576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,8191,0.3462080160776774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,31,0.012533333152532578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,16383,0.6537226835886637
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,63,0.012341332932313284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,127,0.01257066677014033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,15,0.01249066616098086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,255,0.017968000223239262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,127,0.7406346797943115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1023,0.019285333653291065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,511,0.018207999567190807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,4095,0.024671999116738636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,8191,0.031632001201311745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,2047,0.02063999945918719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,16383,0.046122665206591286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,32767,0.051226665576299034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1,0.01227733368674914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,65535,0.06618666648864746
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,3,0.012437333663304647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,7,0.022517333428064983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,15,0.012565333396196365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,127,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,63,0.01250133290886879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,255,0.017978666971127193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,31,0.012453333785136541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,511,0.018298666924238205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1023,0.019167999426523846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,2047,0.021045332153638203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,4095,0.02478933334350586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,16383,0.03938133269548416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,8191,0.03420799970626831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,32767,0.05529599885145823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,65535,0.08085333307584126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1,0.012527999778588613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,3,0.012522666404644648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,15,0.01268799975514412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,31,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,7,0.012741333494583765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,63,0.012858666479587555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,127,0.012757333616415659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,255,0.0183146670460701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1023,0.019493332753578823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,511,0.018805333723624546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,2047,0.02165866643190384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,4095,0.02865600089232127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,8191,0.03299200038115183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,16383,0.04459733267625173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,32767,0.06844266752401988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,7,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,65535,0.1071573297182719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,3,0.012517333030700684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1,0.012554666648308435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,15,0.012602667013804117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,31,0.012442667037248611
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,63,0.012650666137536367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,255,0.01806933308641116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1023,0.019402666638294857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,127,0.012576000144084295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,511,0.01833600054184596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,2047,0.020917333662509918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,8191,0.03477333237727483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,16383,0.03928533444801966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,4095,0.02454400062561035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1,0.017759999881188076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,65535,0.07914666831493378
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,32767,0.05793066819508871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,3,0.018272000054518383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,7,0.01863466699918111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,15,0.018533332894245785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,31,0.01785600061217944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,63,0.018496000518401463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,127,0.01851733277241389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,255,0.02179199953873952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1023,0.023168000082174938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,511,0.0222080002228419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,2047,0.02502399931351344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,4095,0.03146666785081228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,8191,0.036559998989105225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,16383,0.05083199838797251
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1,0.01841066653529803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,7,0.018565333137909572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,65535,0.11874133348464966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,3,0.01836266616980235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,15,0.018533332894245785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,31,0.018426666657129925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,63,0.018629333625237148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,127,0.018629333625237148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,32767,0.0782293329636256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,255,0.021935999393463135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,2047,0.028757333755493164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,511,0.022341333329677582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1023,0.023711999257405598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,4095,0.03342399994532267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,8191,0.044719999035199486
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,16383,0.07373333474000295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,32767,0.1106773316860199
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,65535,0.1943733294804891
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1,0.013034666577974955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,3,0.01340266689658165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,7,0.01322666679819425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,15,0.013493333011865616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,63,0.013450667262077332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,31,0.01332266628742218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,127,0.013424000392357508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,255,0.022970666488011677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,511,0.027263998985290527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,8191,0.18688533703486124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1,0.0222080002228419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1023,0.03952533255020777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,2047,0.06225599845250448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,3,0.022384000321229298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,4095,0.1065013309319814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,7,0.022517333428064983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,7,0.012879999975363413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,15,0.022458667556444805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,127,0.023045333723227184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,31,0.022266666094462078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,511,0.049285332361857094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,63,0.022976001103719074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1023,0.07092800239721934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,255,0.02735999971628189
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,4095,0.19288533926010132
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,8191,0.3743679920832316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1,0.026933332284291584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,3,0.026890667776266735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,2047,0.11620266238848369
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,31,0.027210667729377747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,15,0.027162666122118633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,7,0.027119999130566914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,127,0.027999999622503918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,63,0.02739199995994568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,255,0.0450133333603541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,3,0.01250133290886879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1,0.012485332787036896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,511,0.0717386653025945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,2047,0.22292800744374594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,15,0.012693333129088083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,7,0.012805332740147909
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,31,0.01251199965675672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,127,0.012650666137536367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,255,0.0179626668492953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,511,0.018581333259741466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1023,0.019519999623298645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,63,0.012682666381200155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1023,0.12158399820327759
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,8191,0.8723413149515787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,4095,0.028362666567166645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,8191,0.033258666594823204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,2047,0.021520001192887623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,16383,0.04604800045490265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,32767,0.07021866738796234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1,0.0183999997874101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,7,0.018592000007629395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,3,0.018197332819302876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,15,0.01846933364868164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,31,0.018351999421914417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,63,0.018197332819302876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,127,0.018250666558742523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,4095,0.44576001167297363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,255,0.021935999393463135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,511,0.022240000466505688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1023,0.023658665517965954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,4095,0.0334346666932106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,65535,0.11583999792734782
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,2047,0.02829333394765854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,8191,0.046762665112813316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,32767,0.11555199821790059
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1,0.01858666663368543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,3,0.018453333526849747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,7,0.018858666221300762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,31,0.018474667022625606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,15,0.01863466699918111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,65535,0.1991999944051107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,16383,0.07329066594441731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,63,0.018677332748969395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,127,0.01871466636657715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,255,0.022111999491850536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,2047,0.03145066648721695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1023,0.02712533374627431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,511,0.022783999641736347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,4095,0.043136000633239746
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,8191,0.06628266473611195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,16383,0.10665599505106609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,32767,0.1893120010693868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,65535,0.34409066041310626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1,0.015775999675194424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,3,0.016165333489576977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,7,0.016186666985352833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,63,0.016554666062196095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,31,0.01623999948302905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,15,0.0161920003592968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,127,0.016693333784739178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,255,0.021530665457248688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1,0.027056001126766205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,511,0.04716266691684723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,3,0.026928000152111053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,7,0.02720533311367035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1023,0.06643733382225037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,15,0.027610667049884796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,2047,0.11060800155003865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,31,0.027471999327341717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,63,0.027477333943049114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,4095,0.18986666202545166
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,127,0.028021333118279774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,255,0.04550399879614512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,511,0.0730453332265218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1023,0.12484799822171529
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,4095,0.42400534947713214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,3,0.03418133407831192
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,7,0.03454400102297465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1,0.03396799912055334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,2047,0.2266133427619934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,15,0.03472000112136205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,63,0.0345920001467069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,31,0.03454400102297465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,127,0.044122666120529175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,255,0.06671999891599019
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1023,0.20613867044448853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,511,0.11097066601117452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,2047,0.35647467772165936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,4095,0.7381386756896973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1,0.020768000433842342
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,3,0.020831999679406483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,7,0.021029333273569744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,127,0.022074667116006214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,63,0.021210665504137676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,31,0.021066665649414062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,255,0.04022933294375738
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,15,0.021226666867733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,511,0.06645333270231883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1,0.03505066782236099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1023,0.11617066462834676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,3,0.03515200068553289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,7,0.03555200000603994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,15,0.0356480007370313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,63,0.0363520011305809
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,31,0.03555200000603994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,127,0.04580800235271454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,255,0.06857599814732869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,511,0.11424000064531963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1,0.05221333106358846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,3,0.052933335304260254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,7,0.05352533360322317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,15,0.053599998354911804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1023,0.19435733556747437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,31,0.053472002347310386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,63,0.06075733403364817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,127,0.06955199937025706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,255,0.11725866794586182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,511,0.1918826699256897
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,1,0.027552001178264618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1023,0.3713279962539673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,7,0.02807466685771942
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,3,0.027786667148272198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,15,0.02827200045188268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,31,0.028213332096735638
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,63,0.02844800055027008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,127,0.03990933299064636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,255,0.06097066899140676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,1,0.05551466842492422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,3,0.05570133527119955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,7,0.05648000041643778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,511,0.1065120001633962
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,15,0.056458666920661926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,31,0.057130664587020874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,63,0.06393066545327504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,127,0.07585066556930542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,1,0.08891200025876363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,255,0.12010133266448975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,7,0.09095999598503113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,511,0.19659199317296347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,3,0.08980799714724223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,3,0.012682666381200155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,15,0.09246933460235596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,7,0.012709333250919977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,15,0.012639999389648438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,31,0.012815999488035837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,31,0.09641599655151367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,63,0.01249066616098086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,127,0.012837332983811697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1,0.012671999633312225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,511,0.01883200059334437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1023,0.020261333634455998
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,2047,0.025407999753952026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,255,0.01870399961868922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,63,0.09822932879130046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,4095,0.029839999973773956
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,8191,0.04138133426507314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,16383,0.06451733410358429
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,32767,0.10913599530855815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,3,0.018506667266289394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1,0.0184906671444575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,7,0.018906666586796444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,15,0.018837332725524902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,65535,0.19362666209538779
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,31,0.018853332847356796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,63,0.01874133323629697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,127,0.01882133384545644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,255,0.02236266682545344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,511,0.022874665757020313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,127,0.11681066950162251
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1023,0.027093333502610523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,4095,0.04475200176239014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,2047,0.03192000091075897
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,8191,0.06742933392524719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,16383,0.11211733023325603
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1,0.018768000106016796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,3,0.018485333770513535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,65535,0.3539466857910156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,7,0.01883200059334437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,15,0.01921066641807556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,255,0.19928000370661417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,32767,0.19389333327611288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,31,0.018687999496857326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,63,0.019002666076024372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,127,0.019018666197856266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,255,0.022842665513356526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,511,0.02643733223279317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,2047,0.041984001795450844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,4095,0.06763733426729839
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,8191,0.10644267002741496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,16383,0.18589866161346436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,32767,0.34279998143513996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1023,0.03092266619205475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,65535,0.6550933519999186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,511,0.3572053511937459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,1,0.04557866851488749
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,7,0.04659733176231384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,3,0.04595200220743815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,15,0.04664533336957296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,31,0.04829333225886027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,63,0.05632533133029938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,127,0.06657599906126659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,255,0.11154133081436157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,1,0.09388800462086995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,3,0.09451199571291606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,15,0.10308800141016643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,7,0.09896533687909444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,31,0.10498666763305664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,63,0.1097813347975413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,127,0.13114666938781738
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,255,0.21384000778198242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,1,0.1595200002193451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,7,0.17193067073822021
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,3,0.16235733032226562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,15,0.172106663386027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,31,0.17207467555999756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,63,0.17733865976333618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,127,0.21434134244918823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,255,0.39663465817769367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,1,0.08065600196520488
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,3,0.08401067058245341
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,7,0.0902346670627594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,15,0.0934933324654897
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,31,0.09175466497739156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,63,0.09557867050170898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,127,0.12569066882133484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,1,0.18043200174967447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,3,0.186298668384552
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,7,0.19173866510391235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,15,0.19222933053970337
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,31,0.19115199645360312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,63,0.19490132729212442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,1,0.31702399253845215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,127,0.24424533049265543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,7,0.013061333447694778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,15,0.013162666310866674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,3,0.012815999488035837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,31,0.013093333691358566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,63,0.012784000486135483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,127,0.013114667187134424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,255,0.018522666146357853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,3,0.3285226623217265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,511,0.019146667172511418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1023,0.023589332898457844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,2047,0.02804800122976303
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,8191,0.06251733501752217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,4095,0.040922666589419045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,16383,0.10839999715487163
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,32767,0.18651199340820312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1,0.018981333822011948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,3,0.018954666952292126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,15,0.019066666563351948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,7,0.018863999595244724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,31,0.019050666441520054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,63,0.01905599981546402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,127,0.0189280000825723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,255,0.022677332162857056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,511,0.02643200010061264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1023,0.030805334448814392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,2047,0.04218133290608724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,4095,0.06740266581376393
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,8191,0.11005333065986633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,16383,0.19083199898401895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1,0.019066666563351948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,3,0.01921066641807556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,32767,0.3495146830876668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,7,0.019343999524911244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,15,0.019813333948453266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,31,0.01937066639463107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,63,0.019440000255902607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,127,0.01956266661485036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,255,0.026191999514897663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,511,0.03057066599527995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1023,0.04329599936803182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,2047,0.06623466809590657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,4095,0.10877866546312968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,8191,0.1875093380610148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,7,0.33021867275238037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,16383,0.343610684076945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,3,0.013167999684810638
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,32767,0.7142986456553141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,7,0.013013333082199097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,15,0.013130666067202887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,31,0.01310933381319046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,63,0.013125333935022354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,127,0.013232000172138214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,255,0.018805333723624546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,511,0.022821334501107533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1023,0.02738133321205775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,2047,0.0383146678407987
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,15,0.3288853367169698
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,4095,0.06193066636721293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,8191,0.10586133599281311
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,3,0.01950399950146675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,16383,0.19451733430226645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,32767,0.34110931555430096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,31,0.33060266574223834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1,0.01966933285196622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,7,0.019760000209013622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,15,0.019578666736682255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,63,0.019466667125622433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,127,0.020053333292404812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,31,0.01945066700379054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,255,0.026133333643277485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,511,0.03057066599527995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,4095,0.11002133289972942
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,8191,0.18722132841746011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1023,0.04713066418965658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,16383,0.350383996963501
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1,0.022175999979178112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,2047,0.06600533425807953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,3,0.02216000109910965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,63,0.3326080044110616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,32767,0.6683039665222168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,7,0.022202665607134502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,15,0.022287999590237934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,31,0.022277332842350006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,127,0.022511998812357586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,255,0.02752533306678136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,2047,0.10985066493352254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,511,0.04738133152325948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,63,0.02253866692384084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1023,0.06798399984836578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,8191,0.34252798557281494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,3,0.01811733345190684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1,0.01829333355029424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,16383,0.7123520374298096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,15,0.01858666663368543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,127,0.41467734177907306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,31,0.01848000039656957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,7,0.01855466639002164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,4095,0.18904000520706177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,63,0.018751999984184902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,32767,1.386122703552246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1023,0.11154133081436157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,127,0.019434666881958645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,511,0.0614879975716273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,2047,0.22438400983810425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,255,0.03480533262093862
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,8191,0.8049973646799723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,7,0.01251199965675672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1,0.012266666938861212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,15,0.012175999581813812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,63,0.0124746672809124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,31,0.012426666915416718
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,3,0.012341332932313284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,127,0.012560000022252401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,255,0.01728533332546552
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,511,0.01820266619324684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,16383,1.7108906110127766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1023,0.018933333456516266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,2047,0.020928000410397846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,4095,0.024127999941507976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,8191,0.03030399978160858
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,16383,0.045909335215886436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,32767,0.05031466484069824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,65535,0.069733331600825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,131071,0.09103999535242717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,3,0.011999999483426413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1,0.012357333054145178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,7,0.012533333152532578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,15,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,63,0.012485332787036896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,255,0.017898666361967724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,127,0.012117333710193634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,31,0.012527999778588613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,511,0.01841066653529803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,2047,0.020469332734743755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1023,0.019589333484570186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,8191,0.03432533393303553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,4095,0.024165332317352295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,32767,3.418602625528971
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,16383,0.0386613334218661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,4095,0.40814932187398273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,32767,0.05373333394527435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1,0.01240533341964086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,65535,0.07934933404127757
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,7,0.012746666868527731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,131071,0.12098667025566101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,3,0.012554666648308435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,31,0.01240533341964086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,15,0.012800000607967377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,63,0.012533333152532578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,127,0.012981332838535309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,255,0.018090666582187016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,511,0.018650667121013004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,2047,0.021040000021457672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1023,0.019391999890406925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,4095,0.027679999669392902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,16383,0.04549333453178406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,32767,0.06777599950631459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,8191,0.03249066571394602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,65535,0.10805333654085796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1,0.01090666651725769
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,3,0.010858666151762009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,131071,0.19166932503382364
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,7,0.01109333336353302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,63,0.011114666859308878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,127,0.01116266722480456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,15,0.011237333218256632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1023,0.031541332602500916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,31,0.011087999989589056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,255,0.01431999976436297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,511,0.019754666835069656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,2047,0.05461333195368449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,4095,0.10169066985448201
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,8191,0.19826134045918783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,16383,0.3854453166325887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1,0.012383999923865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,3,0.012383999923865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,32767,0.7644853591918945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,15,0.012826666235923767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,7,0.012698666503032049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,31,0.012597333639860153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,63,0.012602667013804117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,65535,1.5136106808980305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,127,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,255,0.0179626668492953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,511,0.01852799952030182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1023,0.01930133377512296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,2047,0.02072000006834666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,4095,0.02513599892457326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,8191,0.03453866640726725
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,16383,0.0391839991013209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,32767,0.056234667698542275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,65535,0.0817333310842514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,131071,0.12689600388209024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,131071,3.016325314839681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1,0.017701332767804463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,3,0.017786666750907898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,7,0.018197332819302876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,15,0.018138666947682697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,31,0.017946666727463405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,63,0.017877332866191864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,255,0.021850667893886566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1023,0.02329600105683009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,127,0.018309333672126133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,511,0.022229333718617756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,2047,0.024800000091393787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,4095,0.03136000037193298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,8191,0.036101333796978
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,16383,0.053082664807637535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,32767,0.07459733386834462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,65535,0.12455999851226807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1,0.01825599993268649
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,3,0.01820266619324684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,7,0.01828266680240631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,131071,0.21063466866811117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,31,0.01859733338157336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,15,0.01851733277241389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,127,0.01821333294113477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,63,0.01809599995613098
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,255,0.0220320001244545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,511,0.02232533444960912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,2047,0.02847466617822647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,4095,0.03297066688537598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1023,0.02369066576162974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,8191,0.049082666635513306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,16383,0.07190399865309398
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,32767,0.10982933640480042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1,0.010826667149861654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,3,0.011066666493813196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,65535,0.20837332804997763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,131071,0.34798399607340497
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,63,0.011343999455372492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,15,0.011194666226704916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,31,0.011002667248249054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,127,0.011247999966144562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,7,0.011205332974592844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,255,0.014325333138306936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,511,0.020026666422684986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,2047,0.054805333415667214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1023,0.0317493329445521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,4095,0.10291733344395955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,8191,0.19827200969060263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,16383,0.3882559935251872
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,32767,0.7671679655710856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1,0.013077333569526672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,65535,1.5143839518229167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,3,0.013013333082199097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,7,0.013274667163689932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,15,0.013359999905029932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,31,0.013669333110253016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,131071,3.0171146392822266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,127,0.013530666629473368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,63,0.013418667018413544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,255,0.02252800017595291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,511,0.02735466758410136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1023,0.040976000328858696
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,8191,0.18564265966415405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,16383,0.3650933504104614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1,0.021776000658671062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,3,0.022181332111358643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,15,0.022463999688625336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,7,0.022266666094462078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,2047,0.06204266846179962
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,63,0.02223466585079829
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,31,0.022266666094462078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,4095,0.11009066303571065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,127,0.022890667120615642
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1023,0.06931733091672261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,255,0.02758399893840154
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,4095,0.19324266910552979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,511,0.0495306650797526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1,0.026789332429567974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,2047,0.11601600050926208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,7,0.027349332968393963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,15,0.02717333287000656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,8191,0.3509813149770101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,31,0.026965332527955372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,63,0.027461332579453785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,3,0.026906666656335194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,255,0.04363200068473816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,2047,0.22240533431371054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,511,0.0717439999183019
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,127,0.02756800005833308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,4095,0.4171573321024577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,8191,0.8742079734802246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1,0.024714666108290356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,16383,0.711738665898641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1023,0.1202186644077301
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,3,0.024933333198229473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,15,0.02532266577084859
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,7,0.02508266766866048
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,31,0.025274666647116344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,63,0.025514667232831318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,255,0.05402133365472158
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,16383,1.7197813987731934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1023,0.179802676041921
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,2047,0.3427040179570516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1,0.012304000556468964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,3,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,7,0.01251199965675672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,511,0.0995306670665741
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,15,0.012634667257467905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,4095,0.6694133281707764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,31,0.012778667112191519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,63,0.012752000242471695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,127,0.03346133232116699
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,255,0.018112000077962875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1023,0.019738666713237762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,2047,0.021536000072956085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,511,0.018506667266289394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,4095,0.028368001182874043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,8191,0.03311466674009959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,8191,1.4300799369812012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,16383,0.04919999837875366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,127,0.012906666845083237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,32767,0.07320000231266022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,65535,0.12113066514333089
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1,0.018229333062966663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,3,0.01809599995613098
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,7,0.018373332917690277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,15,0.018661333868900936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,131071,0.19344000021616617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,31,0.018266666680574417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,63,0.01868266612291336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,127,0.018458666900793713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,511,0.022384000321229298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,255,0.021701333423455555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1023,0.023567999402681988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,2047,0.02846933404604594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,4095,0.03330666571855545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,8191,0.04994666576385498
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,16383,0.07298133273919423
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,32767,0.11311999956766765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1,0.01838933303952217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,3,0.018298666924238205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,7,0.01870399961868922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,15,0.018592000007629395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,65535,0.20964799324671426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,131071,0.3604960044225057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,31,0.01863466699918111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,63,0.018506667266289394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,127,0.018816000471512478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,255,0.022128000855445862
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,2047,0.03199466566244761
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1023,0.02700799951950709
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,511,0.022709332406520844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,4095,0.0432640016078949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,8191,0.06593066453933716
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,16383,2.8466933568318686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,16383,0.10643200079600017
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,32767,0.1907306710879008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1,0.01119999960064888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,3,0.011109333485364914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,15,0.011472000430027643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,65535,0.34357865651448566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,31,0.011312000453472137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,63,0.011312000453472137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,127,0.011183999478816986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,255,0.014661333213249842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,511,0.020165332903464634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1023,0.03194133440653483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,131071,0.7152106761932373
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,2047,0.05751466751098633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,4095,0.1050933301448822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,7,0.01126933346192042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,8191,0.19962133963902792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,16383,0.38972798983256024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,32767,0.7678559621175131
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,65535,1.527616024017334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1,0.01599466676513354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,3,0.01586666703224182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,7,0.016250666230916977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,15,0.016154666741689045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,31,0.015935999651749928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,131071,3.0637973149617515
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,63,0.016410666207472484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,127,0.01670933390657107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,255,0.021216000119845074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,511,0.045168002446492515
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1023,0.06491200129191081
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1,0.02677333354949951
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,7,0.02699733277161916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,8191,0.36458667119344074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,3,0.027024000883102417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,15,0.027119999130566914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,2047,0.11040000120798747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,4095,0.19602133830388388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,63,0.027210667729377747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,127,0.028186666468779247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,31,0.026922665536403656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,255,0.04427200059096018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,511,0.07246399919191997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1023,0.12337066729863484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,2047,0.22598934173583984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,3,0.033941333492596946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1,0.033514666060606636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,7,0.03385066737731298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,8191,0.8660639921824137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,4095,0.42371201515197754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,15,0.03403733422358831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,127,0.04297066728274027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,255,0.06559466818968455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,31,0.034314667185147606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,2047,0.35611732800801593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,511,0.11040533582369487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,63,0.03472000112136205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,4095,0.7388160228729248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1,0.040789333482583366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,7,0.04162666698296865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,3,0.041290665666262306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1023,0.1928159991900126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,31,0.04192533095677694
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,8191,1.4482933680216472
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,63,0.04630400240421295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,15,0.04161066561937332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,127,0.05294933418432871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,2047,0.625162680943807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,255,0.10178132851918538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1023,0.3253866632779439
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,4095,1.3473706245422363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,511,0.18769599994023642
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1,0.020714666694402695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,3,0.020453333854675293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,7,0.02082666630546252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,31,0.020874666670958202
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,15,0.020901332298914593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,63,0.021157334248224895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,255,0.03843733419974645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,8191,2.6785014470418296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,127,0.021615999440352123
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,511,0.06579733391602834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1,0.034160000582536064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,3,0.034272000193595886
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,7,0.034416000048319496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1023,0.11614400148391724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,31,0.034517332911491394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,2047,0.21433599789937338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,63,0.03501333296298981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,15,0.03512533257404963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,127,0.04464533428351084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,255,0.06785066425800323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,511,0.11190399527549744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1,0.05227200190226237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,3,0.05232533315817515
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,7,0.05277333160241445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1023,0.19606399536132812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,2047,0.36276264985402423
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,15,0.052501335740089417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,31,0.0528053343296051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,63,0.05903466542561849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,127,0.06830933193365733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1,0.07264000177383423
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,3,0.07391466697057088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,255,0.11505066355069478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,511,0.1895893414815267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,7,0.07583466668923695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1023,0.34198399384816486
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,15,0.07592533528804779
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,2047,0.6452533404032389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,31,0.07835199932257335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,127,0.09920533498128255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,255,0.19056532780329385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,63,0.0798933357000351
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1023,0.632144014040629
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,2047,1.355855941772461
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,511,0.3570559819539388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1,0.027285332481066387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,3,0.027269333600997925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,31,0.027989332874615986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,7,0.02775999903678894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,15,0.027855999767780304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,63,0.02844800055027008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,127,0.040463998913764954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,255,0.061205332477887474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,511,0.10442666212717693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1,0.053642665346463524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1023,0.184879998366038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,3,0.053685332338015236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,7,0.05422399938106537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,15,0.054197331269582115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,31,0.05423999826113383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,63,0.06150933106740316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,127,0.07191466788450877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,511,0.1948053240776062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,255,0.11777066191037495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1,0.08682133754094441
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,3,0.08700266480445862
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,7,0.08861333131790161
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,15,0.08986666798591614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1023,0.3505866527557373
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,31,0.09397866328557332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,127,0.11230400204658508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,63,0.09615466992060344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,255,0.19554134209950766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1,0.13613333304723105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,7,0.14013866583506265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,511,0.3485013246536255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,31,0.14390933513641357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,15,0.14697066942850748
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,3,0.012416000167528788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,7,0.012778667112191519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1023,0.7069653669993082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,15,0.012826666235923767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,63,0.14461333552996317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,63,0.012693333129088083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,255,0.018426666657129925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,127,0.1784213383992513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,511,0.01884799947341283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1023,0.02013333390156428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,3,0.13683733344078064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,4095,0.029930666089057922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1023,1.2396693229675293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,8191,0.043162668744723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,16383,0.06597866614659627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,127,0.013013333082199097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,65535,0.19369065761566162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,32767,0.11499733726183574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1,0.018511999398469925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,131071,0.3511626720428467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,3,0.018426666657129925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,7,0.018805333723624546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,15,0.018687999496857326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,31,0.018661333868900936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,63,0.01828266680240631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,255,0.022341333329677582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,127,0.018768000106016796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,511,0.023061332603295643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1023,0.026949333647886913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,4095,0.04663999875386556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,2047,0.03175999969244003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,8191,0.06675200164318085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,16383,0.11122666796048482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,2047,0.02496533344189326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,32767,0.19246933857599893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,65535,0.3492159843444824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1,0.018650667121013004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,3,0.018645333747069042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,7,0.018981333822011948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,15,0.018687999496857326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,131071,0.6707359949747721
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,31,0.01894933357834816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,63,0.018944000204404194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,127,0.018885333091020584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,511,0.02629333237806956
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,255,0.022805333137512207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1023,0.030965333183606465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,2047,0.04273599882920583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,4095,0.06735999882221222
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,8191,0.11435199777285258
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,16383,0.18752533197402954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,32767,0.34324268500010174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,255,0.3612213134765625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1,0.011157333850860596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,3,0.01128000020980835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,65535,0.6559040149052938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,15,0.011306667079528173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,511,0.6934506893157959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,31,0.011343999455372492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,63,0.011551999797423681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,127,0.011648000528415045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,255,0.014965333044528961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1023,0.035642666121323906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,131071,1.394261360168457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,2047,0.06029333174228668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,4095,0.10866133371988933
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,8191,0.2044853369394938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,31,0.012762666990359625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,16383,0.407039999961853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,32767,0.7770666281382242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,65535,1.5891040166219075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,7,0.011546666423479715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,131071,3.1617867151896157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,511,0.0205226664741834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,1,0.04496000210444132
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,3,0.045184001326560974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,15,0.04606399933497111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,7,0.04594666759173075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,31,0.04692266881465912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,63,0.054325332244237266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,127,0.06569066643714905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,255,0.10876799623171489
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,1,0.09066667159398396
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,3,0.09150399764378865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,511,0.18589866161346436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,7,0.09335999687512715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,15,0.09702933828035991
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,31,0.10028800368309021
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,63,0.10517866412798564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,127,0.12568533420562744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,255,0.20709866285324097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,1,0.1550986667474111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,3,0.15732799967130026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,7,0.16422933340072632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,511,0.35981865723927814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,15,0.16840000947316489
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,31,0.16898133357365927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,63,0.17313067118326822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,127,0.20510933796564737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,1,0.26057066520055133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,7,0.28044267495473224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,255,0.36510932445526123
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,15,0.2794559995333354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,3,0.26637333631515503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,31,0.27375467618306476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,511,0.7237866719563802
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,63,0.2807946602503459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,127,0.3728799819946289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,255,0.6520746548970541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,1,0.07902400195598602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,511,1.367429256439209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,3,0.08050666749477386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,7,0.08461866776148479
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,31,0.09000533819198608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,15,0.08903466661771138
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,63,0.09177600344022115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,127,0.11947199702262878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,3,0.1702880064646403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,255,0.19430400927861533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,7,0.1778506636619568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,1,0.16474133729934692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,15,0.18124266465504965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,31,0.18129066626230875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,63,0.18392000595728555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,127,0.22952000300089517
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,1,0.2993493278821309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,3,0.31333333253860474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,255,0.38553067048390705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,15,0.3179786602656047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,7,0.3200586636861165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,31,0.3206719954808553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,63,0.32264532645543414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,127,0.39948264757792157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,1,0.5187733173370361
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,255,0.757584015528361
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,3,0.5365226666132609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,7,0.5422506729761759
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,3,0.012928000340859095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,7,0.013050666699806849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,31,0.013264000415802002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,63,0.013077333569526672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,15,0.012917333592971167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,15,0.546175996462504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,127,0.01302933320403099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,255,0.018266666680574417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,511,0.019296000401178997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,2047,0.028143999477227528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,31,0.5490080118179321
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,4095,0.03937066594759623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,8191,0.06259199976921082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,32767,0.18626666069030762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,16383,0.10638933380444844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1023,0.023546665906906128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1,0.01844266677896182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,3,0.018960000326236088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,7,0.018901333212852478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,31,0.019098666807015736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,15,0.018698666244745255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,63,0.01882133384545644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,127,0.019082666685183842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,255,0.02248000105222066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,511,0.026191999514897663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1023,0.030997333427270252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,2047,0.043023998538653054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,4095,0.06716266771157582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,8191,0.10946133732795715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,16383,0.18704533576965332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,32767,0.3497440020243327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,65535,0.3466720183690389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1,0.018458666900793713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,3,0.019141333798567455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,65535,0.6704320112864176
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,7,0.019189332922299702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,15,0.018709332992633183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,31,0.019050666441520054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,63,0.018960000326236088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,127,0.01941866676012675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,255,0.025786665578683216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,511,0.030618667602539062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,2047,0.069733331600825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1023,0.04164800047874451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,4095,0.10820266604423523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,8191,0.19178134202957153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1,0.013823999712864557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,16383,0.34924264748891193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,3,0.014117332796255747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,7,0.014138666292031607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,15,0.014064000298579534
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,31,0.01404800017674764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,32767,0.7183093229929606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,63,0.014416000495354334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,127,0.014746667196353277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,255,0.0191040001809597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1023,0.057018667459487915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,65535,1.3972266515096028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,2047,0.10109866658846538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,4095,0.19937600692113241
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,511,0.032960000137488045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,8191,0.3781333367029826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1,0.012906666845083237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,16383,0.798911968866984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,3,0.012991999586423239
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,7,0.013077333569526672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,32767,1.5790665944417317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,31,0.013157332936922709
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,15,0.013093333691358566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,63,0.013077333569526672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,63,0.5599679946899414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,127,0.013167999684810638
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,255,0.018794666975736618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,511,0.02229333420594533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,65535,3.1550347010294595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1023,0.02700799951950709
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,8191,0.10615467031796773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,2047,0.04093866546948751
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,16383,0.19618133703867593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1,0.019007999449968338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,4095,0.06091199815273285
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,7,0.018624000251293182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,15,0.018810667097568512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,65535,0.6569706598917643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,3,0.01884799947341283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,63,0.01852799952030182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,127,0.019173332800467808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,255,0.025802666942278545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,511,0.03053866575161616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,31,0.018730666488409042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,4095,0.1179039975007375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1023,0.04535999894142151
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,16383,0.3475946585337321
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,8191,0.19205333789189658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,2047,0.06646933158238728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1,0.014042666802803675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,32767,0.7107840379079183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,7,0.014240000396966934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,3,0.014378666877746582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,15,0.01414399966597557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,31,0.014256000518798828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,32767,0.3400213321050008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,63,0.0143306665122509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,255,0.019365333020687103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,127,0.014794666320085526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,511,0.03209066639343897
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,127,0.6842559973398844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,2047,0.10222933689753215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,65535,1.3872853914896648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,4095,0.20149334271748862
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1023,0.058320000767707825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1,0.012245333443085352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,8191,0.3843199809392293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,7,0.044624000787734985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,3,0.012250666817029318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,15,0.012229333321253458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,16383,0.7564266522725424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,127,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,63,0.012298667182525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,31,0.012794667234023413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,255,0.017797333498795826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,511,0.01798933371901512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,8191,0.03033066789309184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,32767,1.5954079627990723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,2047,0.0205226664741834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1023,0.018565333137909572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,4095,0.024282666544119518
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,16383,0.044362664222717285
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1,0.012304000556468964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,65535,0.062122667829195656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,131071,0.08957333366076152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,3,0.012261333564917246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,32767,0.04975999891757965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,7,0.012586666891972223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,63,0.01246400053302447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,31,0.012159999459981918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,15,0.01251199965675672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,127,0.012576000144084295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,255,0.01729600007335345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,511,0.017984000345071156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1023,0.01877333347996076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,2047,0.021061333517233532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,8191,0.033957332372665405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,4095,0.023925334215164185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,32767,0.05365333457787832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,16383,0.038431999584039055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,65535,0.07850666840871175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1,0.010863999525705973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,131071,0.12014933427174886
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,7,0.011039999624093374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,3,0.010816000401973724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,15,0.010645333677530289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,63,0.011045332998037338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,31,0.010879999647537867
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,255,0.014181333283583323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,127,0.011039999624093374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,511,0.01972266659140587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1023,0.031354665756225586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,2047,0.05459199845790863
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,4095,0.10135466853777568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,8191,0.1943626602490743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,16383,0.384117325146993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,32767,0.7650187015533447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,3,0.012527999778588613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1,0.012416000167528788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,7,0.01246400053302447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,15,0.01267733300725619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,31,0.012634667257467905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,63,0.012522666404644648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,127,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,65535,1.5160373051961262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,255,0.017845333864291508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,511,0.018496000518401463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1023,0.01916266605257988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,8191,0.03387733300526937
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,2047,0.02075733368595441
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,4095,0.023770667612552643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,16383,0.03908266623814901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,32767,0.05226666728655497
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,131071,3.0171839396158853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,65535,0.07897600034872691
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,3,0.017925333231687546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1,0.017237332959969837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,131071,0.11806399623552959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,7,0.017946666727463405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,15,0.018394666413466137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,31,0.017925333231687546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,63,0.018058666338523228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,255,0.021664001047611237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,127,0.01803733284274737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,511,0.02199466774861018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1023,0.022970666488011677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,2047,0.025205334027608235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,4095,0.031178665657838184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,8191,0.035936000446478523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,16383,0.04881600042184194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,32767,0.07355733215808868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1,0.010901333143313726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,65535,0.1169546643892924
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,3,0.010762666662534079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,7,0.011050666371981302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,131071,0.19792000452677408
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,15,0.011488000551859537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,31,0.011141333729028702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,63,0.010928000013033548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,127,0.011141333729028702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,255,0.014335999886194864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,511,0.019839999576409657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1023,0.03133866687615713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,2047,0.05451733370621999
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,4095,0.10130133231480916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,8191,0.1967359979947408
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,65535,2.9965759913126626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,16383,0.38625065485636395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,32767,0.7608693440755209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1,0.01313599944114685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,65535,1.5128107070922852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,3,0.012991999586423239
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,15,0.013301332791646322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,7,0.013354666531085968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,31,0.013093333691358566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,131071,3.0284268061319985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,63,0.013194666554530462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,127,0.013434667140245438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,255,0.022143999735514324
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,511,0.026874666412671406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1023,0.03736533224582672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,2047,0.06035199761390686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,8191,0.18339733282725015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,16383,0.33887465794881183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1,0.021898667017618816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,4095,0.10473066568374634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,7,0.021962667504946392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,3,0.022085333863894146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,15,0.02205866575241089
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,31,0.022005334496498108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,32767,0.6963840325673422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,127,0.022448000808556873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,255,0.027189334233601887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,511,0.04755199948946635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,63,0.02221333235502243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1023,0.06885866820812225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,8191,0.3717546860376994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1,0.018266666680574417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,4095,0.20097599426905313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,16383,0.7139999866485596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,3,0.018458666900793713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,7,0.018618666877349217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,15,0.018613333503405254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,255,1.4054986635843914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,31,0.01882133384545644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,63,0.019050666441520054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,127,0.019333332777023315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,2047,0.12005333105723064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,32767,1.386288007100423
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,255,0.0324799989660581
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,511,0.06117866436640421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,2047,0.21185600757598877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1,0.012383999923865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,4095,0.40828267733256024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,3,0.012351999680201212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,7,0.04147200038035711
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,15,0.012469333906968435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,31,0.012432000289360682
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,63,0.012874666601419449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,255,0.017887999614079792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,127,0.012858666479587555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1023,0.11146666606267293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,511,0.018543999642133713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1023,0.019317333896954853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,2047,0.021359999974568684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,4095,0.02775466690460841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,8191,0.03250666707754135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,8191,0.8633386294047037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,32767,0.06923200190067291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,16383,1.7105973561604817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1,0.01817600056529045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,65535,0.11169067025184631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,3,0.018266666680574417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,7,0.01815466706951459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,131071,0.19101866086324057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,16383,0.04438399771849314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,15,0.01788266624013583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,63,0.018144000321626663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,31,0.018346666047970455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,127,0.018133333573738735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,511,0.02233600119749705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,255,0.021669333179791767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1023,0.023455999791622162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,4095,0.0329066663980484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,8191,0.047082667549451195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,2047,0.027973333994547527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,32767,0.11282133062680562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,16383,0.07443200051784515
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1,0.010885333021481832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,65535,0.195743997891744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,3,0.011157333850860596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,7,0.011136000355084738
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,15,0.011120000233252844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,131071,0.35607465108235675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,31,0.011066666493813196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,63,0.010869332899649939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,127,0.011178666104873022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,255,0.014463999619086584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,511,0.020053333292404812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,2047,0.055018668373425804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1023,0.03169599920511246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,4095,0.10368000467618306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,8191,0.19822933276494345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,16383,0.38604267438252765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,32767,0.7634027004241943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,65535,1.5182347297668457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,3,0.015840000162522
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1,0.015930666277805965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,7,0.015850666910409927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,15,0.01613333324591319
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,131071,3.0419947306315103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,31,0.01589866727590561
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,63,0.016234666109085083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,127,0.01639466608564059
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,255,0.02120000123977661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,511,0.043978666265805565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1023,0.06580266853173573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,8191,0.34381333986918133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1,0.026719999810059864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,2047,0.10860266288121541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,3,0.026816000541051228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,4095,0.18710933128992716
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,7,0.026575999955336254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,16383,0.6980160077412924
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,15,0.027098665634791057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,63,0.02712533374627431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,31,0.027056001126766205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,32767,2.8402134577433267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,255,0.04414933423201243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,511,0.07205333312352498
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,127,0.027653334041436512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1023,0.12276800473531087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,4095,0.44490134716033936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,8191,0.8678239981333414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,3,0.025381334125995636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1,0.02550400048494339
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,2047,0.2251360019048055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,15,0.025978667040665943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,7,0.025786665578683216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,31,0.025583999852339428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,63,0.02611733227968216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,255,0.05397333204746246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,511,0.0990613301595052
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,2047,0.36897067228953045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,16383,1.711418628692627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,127,0.03202133377393087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,4095,0.7263306776682535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1023,0.19324266910552979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1,0.020197333147128422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,31,0.020666666328907013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,8191,1.4359839757283528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,15,0.02067733307679494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,3,0.020448000480731327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,127,0.02146666745344798
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,255,0.036533333361148834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,7,0.020586666961510975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,63,0.021066665649414062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1023,0.11542399724324544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,511,0.0668693333864212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,16383,2.8573280970255532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1,0.033573334415753685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,3,0.03369600077470144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,7,0.03425599883000056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,2047,0.2137813369433085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,31,0.03426666557788849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,15,0.034330666065216064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,4095,0.40915199120839435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,127,0.043237333496411644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,255,0.06575466692447662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1023,0.19196800390879312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,511,0.11158399780591328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,3,0.04308799902598063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,4095,0.7337013085683187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,63,0.034858666360378265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,7,0.04303466777006785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,63,0.048101335763931274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,31,0.04330133398373922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,127,0.05781333148479462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,2047,0.37988801797231037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,255,0.09796266754468282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,15,0.043493335445721946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1,0.04251733422279358
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,2047,0.691648006439209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,511,0.18845866123835245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1023,0.3286133408546448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,4095,1.3564747174580891
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,7,0.02777066578467687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,3,0.027290667096773785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1,0.027119999130566914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,15,0.02777066578467687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,31,0.027653334041436512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,63,0.02811199923356374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,255,0.058864002426465355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1,0.05159999926884969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,511,0.10437867045402527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1023,0.18437333901723227
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,127,0.03684266656637192
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,3,0.05159999926884969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,7,0.0524586687485377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,15,0.052389333645502724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,2047,0.35072000821431476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,31,0.052005335688591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,63,0.06001600126425425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,255,0.11291733384132385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,127,0.06941333413124084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,511,0.19031999508539835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,3,0.07643733421961467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,7,0.07799466451009114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1,0.075354665517807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,31,0.08111999928951263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1023,0.3442026774088542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,15,0.07815466821193695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1,0.01258133351802826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,3,0.012373333175977072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,7,0.012661332885424295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,15,0.012666666259368261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,31,0.012784000486135483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,63,0.012527999778588613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,127,0.012768000364303589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,127,0.09821333487828572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,2047,0.702725330988566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,255,0.01803733284274737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,63,0.08223466575145721
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,2047,0.02479466547568639
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,8191,0.041509332756201424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,4095,0.029418667157491047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,16383,0.06423999865849812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,511,0.018677332748969395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,32767,0.10771733522415161
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,65535,0.19084266821543375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1,0.01810666670401891
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,3,0.018053332964579265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,131071,0.3484799861907959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,255,0.19179733594258627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,7,0.018090666582187016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,15,0.01830400029818217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,31,0.01806933308641116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,63,0.018042666216691334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,255,0.021984001000722248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,511,0.022495999932289124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1023,0.026954665780067444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,2047,0.03150933235883713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,4095,0.043375998735427856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,8191,0.06611733138561249
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,16383,0.11127466956774394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,32767,0.19250667095184326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,511,0.33450667063395184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,65535,0.34886399904886883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1,0.011007999380429586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,15,0.011237333218256632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,7,0.011109333485364914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,131071,0.6715839703877767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,31,0.011264000087976456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,127,0.011354666203260422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,63,0.011087999989589056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,255,0.014389333625634512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,511,0.020186666399240494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1023,0.031557333966096245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,127,0.01841066653529803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,2047,0.05714133381843567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,4095,0.10683733224868774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,16383,0.39000535011291504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,8191,0.19973333676656088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,32767,0.7645599842071533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,2047,1.2461333274841309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,65535,1.533125400543213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,3,0.011136000355084738
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1023,0.6957066853841146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1023,0.020128000527620316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,131071,3.0586560567220054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1,0.04417600234349569
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,3,0.04469866553942362
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,7,0.045226668318112694
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,15,0.045370668172836304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,31,0.045114666223526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,63,0.05236266553401947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,255,0.11053867141405742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,127,0.06242666641871134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1,0.08564800024032593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,511,0.18122132619222006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,3,0.08638933300971985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,7,0.08852266271909077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,15,0.08948799967765808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1023,0.3370720148086548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,31,0.09410132964452107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,63,0.09832533200581868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,255,0.19716266791025797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1,0.13985066612561545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,127,0.11577066779136658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,7,0.1449013352394104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,511,0.37220799922943115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,3,0.14069867134094238
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1023,0.7070933183034261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,63,0.15324800213178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,31,0.1488800048828125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,15,0.1502133309841156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,127,0.19935999313990274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,255,0.366592009862264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,511,0.7003520329793295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1023,1.2450719674428303
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,1,0.07715733349323273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,7,0.08045866588751475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,3,0.07821866869926453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,15,0.08157866696516673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,31,0.08801600337028503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,63,0.08947733044624329
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,127,0.10823999842007954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,1,0.155157337586085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,7,0.16539733608563742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,3,0.15665599703788757
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,511,0.3441653251647949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,15,0.16982932885487875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,255,0.1888266603151957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,31,0.17273066441218057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,63,0.17369065682093301
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,127,0.21547200282414755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,1,0.27029333511988324
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,255,0.39139731725056964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,3,0.27749866247177124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,7,0.2892640034357707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1,0.012757333616415659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,3,0.01292266696691513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,15,0.28861333926518756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,7,0.013072000195582708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,15,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,63,0.013093333691358566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,31,0.01302933320403099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,127,0.012954667210578918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,511,0.019018666197856266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,31,0.29172799984614056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,255,0.0186666672428449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1023,0.022976001103719074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,2047,0.027717334528764088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,4095,0.038736000657081604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,16383,0.10736533006032307
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,65535,0.34405867258707684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,32767,0.18609066804250082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,8191,0.06460799773534139
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,511,0.7178080081939697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1,0.01851733277241389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,3,0.018506667266289394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,15,0.018640000373125076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,7,0.018592000007629395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,131071,0.7008960247039795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,63,0.01848000039656957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,127,0.018522666146357853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,255,0.022533332308133442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,511,0.026133333643277485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1023,0.030752000709374745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,4095,0.06577066580454509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,8191,0.11060800155003865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,2047,0.047168001532554626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,16383,0.18732800086339316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,32767,0.34991466999053955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1,0.011359999577204386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,63,0.2967733343442281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,3,0.011567999919255575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,7,0.0116799995303154
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,65535,0.7146133581797282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,15,0.011509332805871964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,127,0.011834666132926941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,131071,1.387813409169515
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,63,0.011765333513418833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,255,0.01482133318980535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,2047,0.060821334520975746
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1023,0.03573333223660787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,4095,0.11081066727638245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,511,0.020581333587567013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,16383,0.40140267213185626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,8191,0.20572799444198608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,31,0.011525332927703857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,127,0.3635626633961995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,32767,0.8155946731567383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,31,0.019386666516462963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1,0.013162666310866674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,3,0.012874666601419449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,7,0.013232000172138214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,15,0.013151999562978745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,65535,1.6317760149637859
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,255,0.721829334894816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,63,0.012997332960367203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,127,0.013061333447694778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,255,0.018981333822011948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,511,0.02214933435122172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1023,0.02703999976317088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,131071,3.2069387435913086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,2047,0.036730666955312095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,4095,0.06137600044409434
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,16383,0.1842613418896993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1,0.01121066634853681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,32767,0.3622239828109741
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,3,0.011424000064531961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,7,0.011445333560307821
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,31,0.013082666943470636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,15,0.011343999455372492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,65535,0.7006826400756836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,31,0.011407999942700068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,63,0.011663999408483505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,127,0.011733333269755045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,255,0.01481066644191742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,511,0.020453333854675293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1023,0.03544000039498011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,4095,0.1113866666952769
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,131071,1.3621172904968262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,2047,0.060906668504079185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,8191,0.20966933170954385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1,0.012191999703645706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,16383,0.4089333216349284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,3,0.012159999459981918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,32767,0.8288959662119547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,7,0.01231466606259346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,15,0.012378666549921036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,63,0.012416000167528788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,31,0.012362666428089142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,127,0.012256000190973282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,511,0.017738666385412216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,255,0.016976000120242436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,65535,1.630949338277181
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1023,0.01858666663368543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,16383,0.044581333796183266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,4095,0.02309333284695943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,2047,0.020138667275508244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,32767,0.049327999353408813
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,65535,0.06287466486295064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,131071,0.08764800429344177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1,0.010735999792814255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,8191,0.029461334149042766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,3,0.010837333897749582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,7,0.010725333044926325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,15,0.010992000500361124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,63,0.010762666662534079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,127,0.010762666662534079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,31,0.010735999792814255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,131071,3.254879951477051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,255,0.013877333452304205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1023,0.0308746670683225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,511,0.019530666371186573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,2047,0.0543093333641688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,8191,0.10287466645240784
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,4095,0.10077333450317383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,8191,0.1934986710548401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,16383,0.37928001085917157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,32767,0.7592533429463705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1,0.012229333321253458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,7,0.012362666428089142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,3,0.012341332932313284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,15,0.01246400053302447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,65535,1.5253973007202148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,63,0.012346666306257248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,127,0.012362666428089142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,31,0.012154666086037954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,255,0.01739199956258138
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,511,0.01844800015290578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1023,0.01878400022784869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,2047,0.021125334004561108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,131071,3.0220746994018555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,16383,0.03905066599448522
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,4095,0.02418133368094762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,8191,0.033344000577926636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,32767,0.05307200054327647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,65535,0.07720533510049184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,131071,0.11955199639002483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,3,0.010735999792814255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,15,0.011077333241701126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,31,0.0107893335322539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,63,0.011125333607196808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,7,0.010970667004585266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1,0.010970667004585266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,127,0.011061333119869232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,255,0.014181333283583323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1023,0.031189332405726116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,511,0.01964266722400983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,4095,0.10110933581988017
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,2047,0.05434666574001312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,16383,0.3858986695607503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,8191,0.19459732373555502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,511,1.388005256652832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,32767,0.7653280099232992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,3,0.013093333691358566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1,0.013007999708255133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,65535,1.513045310974121
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,7,0.013023999830087027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,15,0.013248000293970108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,131071,3.0161867141723633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,127,0.013365333278973898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,255,0.02187199890613556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,511,0.026672000686327618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1023,0.036357333262761436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,8191,0.1839359998703003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,16383,0.3384693463643392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,2047,0.06368533273537953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1,0.013983999689420065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,4095,0.10309867064158122
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,32767,0.697370688120524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,3,0.014042666802803675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,15,0.01402666668097178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,7,0.014101333916187286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,127,0.014655999839305878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,63,0.014544000228246054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,255,0.019152000546455383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,65535,1.3629652659098308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,511,0.033786666889985405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1023,0.05795733133951823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,31,0.014682666709025701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,4095,0.20476800203323364
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1,0.01257066677014033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,3,0.012565333396196365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,2047,0.10529067118962605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,8191,0.4135146538416545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,7,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,63,0.01257066677014033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,127,0.012608000387748083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,16383,0.8033119837443033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,255,0.018346666047970455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,511,0.018415999909241993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,31,0.012170666207869848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,2047,0.02143999934196472
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,4095,0.0277813325325648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,8191,0.031914666295051575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,15,0.012810666114091873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,32767,1.608565330505371
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1023,0.01911466692884763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,16383,0.04452799757321676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1,0.010773333410422007
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,32767,0.06825066606203715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,65535,0.10970133543014526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,131071,0.19041067361831665
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,3,0.010847999403874079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,31,0.010847999403874079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,15,0.010960000256697336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,63,0.010832000523805618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,7,0.01102399950226148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,127,0.010922666639089584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,511,0.019658666104078293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1023,0.031141333281993866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,255,0.014042666802803675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,2047,0.05428266525268555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,4095,0.10115200281143188
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,8191,0.19526932636896768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,16383,0.3847839832305908
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,65535,3.2072585423787436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,32767,0.7613600095113119
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,65535,1.5250986417134602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1,0.015728000551462173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,3,0.01588800052801768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,7,0.01600533351302147
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,131071,3.0192480087280273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,31,0.01591466615597407
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,63,0.01617066686352094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,127,0.016602666427691776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,255,0.021290667355060577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,511,0.04257600009441376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,63,0.013424000392357508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1023,0.06504533191521962
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,15,0.0161920003592968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,8191,0.36000533898671466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,2047,0.10808533430099487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1,0.018496000518401463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,16383,0.6974506378173828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,3,0.018432000031073887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,7,0.019061333189407986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,15,0.018810667097568512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,4095,0.1916853388150533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,63,0.018885333091020584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,32767,1.3627999623616536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,31,0.018543999642133713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,127,0.01951466624935468
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,255,0.03366400053103765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,2047,0.2267893354098002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1023,0.11404266953468323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,4095,0.4381973346074422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,511,0.06474133332570393
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,8191,0.8560266494750977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,16383,1.6989013353983562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1,0.020037333170572918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,15,0.020746666938066483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,63,0.020794666061798733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,31,0.020586666961510975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,3,0.020224000016848247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,7,0.0206986665725708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,32767,3.4107840855916343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,255,0.034416000048319496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,127,0.021946666141351063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,511,0.0643093337615331
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1023,0.1136799951394399
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,7,0.02565866708755493
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,3,0.02521066615978877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,8191,0.85099196434021
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1,0.02502399931351344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,2047,0.21338133017222086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,15,0.025621332228183746
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,31,0.025546667476495106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,4095,0.43249066670735675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,63,0.025813333690166473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,255,0.058703998724619545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,127,0.033530667424201965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,511,0.09980266292889912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1023,0.1811306675275167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,4095,0.6782026290893555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,8191,1.434127966562907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,2047,0.3480213483174642
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1,0.026880001028378803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,3,0.027024000883102417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,7,0.027637332677841187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,31,0.027471999327341717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,15,0.02755733331044515
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,63,0.027984000742435455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,127,0.038047999143600464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,511,0.10246933499972026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1,0.04244266450405121
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1023,0.19453332821528116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,255,0.058890665570894875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,3,0.04268800218900045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,2047,0.3690026601155599
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,7,0.04307733476161957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,15,0.043162668744723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,31,0.04310933252175649
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,127,0.05714133381843567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,4095,0.7218133608500162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,63,0.04814399778842926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,255,0.10034132997194926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1,0.012400000045696894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,7,0.012608000387748083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,63,0.012709333250919977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,15,0.012560000022252401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,127,0.012719999998807907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,255,0.018330667167901993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,511,0.01841066653529803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,31,0.01351999988158544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1023,0.019727999965349834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,2047,0.024271999796231587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,8191,0.040021332601706185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,4095,0.02905600021282832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,16383,0.06317333380381267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,32767,0.10781866312026978
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,65535,0.18926399946212769
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1,0.011237333218256632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,131071,0.34957865873972577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,3,0.010869332899649939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,511,0.17661333084106445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,15,0.010922666639089584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,7,0.01129066695769628
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,31,0.011045332998037338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,63,0.010960000256697336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,127,0.011141333729028702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,511,0.01989866668979327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1023,0.03143466760714849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,2047,0.054885332783063255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,4095,0.1034399966398875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,8191,0.1986080010732015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1023,0.3545440038045247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,4095,1.3584693272908528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,16383,0.3910026550292969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,32767,0.7701333363850912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,65535,1.5162773132324219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,255,0.01440000037352244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,131071,3.041221300760905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,2047,0.6435519854227701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1,0.04461333155632019
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,7,0.04507733384768168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,3,0.04470400015513102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,15,0.045237332582473755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,31,0.0452159990866979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,63,0.052416001756985985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,127,0.06356266637643178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1,0.0748533308506012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,3,0.07500799993673961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,255,0.10443733135859172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,511,0.18066134055455527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1023,0.3562613328297933
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,15,0.07774933179219563
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,7,0.0775733341773351
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,2047,0.6838080088297526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,31,0.0803413341442744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,63,0.08501332998275757
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,127,0.10308266679445903
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,255,0.1823199987411499
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,511,0.35685332616170246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1023,0.6903946399688721
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,2047,1.356357256571452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,3,0.0776693324247996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,7,0.07979199786980946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1,0.0765066643555959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,15,0.07987200220425923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,31,0.08623466889063518
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,63,0.0883786678314209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,127,0.10453866918881734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1,0.1397760013739268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,255,0.1860533356666565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,3,0.14102400342623392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,511,0.35990933577219647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,7,0.14401066303253174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1023,0.6477973461151123
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,15,0.14985600113868713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,3,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,7,0.012869333227475485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,15,0.012949333836634954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,31,0.012837332983811697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,63,0.012810666114091873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,31,0.1528320014476776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,127,0.012965332716703415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,255,0.018464000274737675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1023,0.023002666731675465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,511,0.019029332945744198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,2047,0.02775466690460841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,4095,0.037765334049860634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,8191,0.06150933106740316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,16383,0.10545600454012553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,32767,0.1862293283144633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,65535,0.36486931641896564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1,0.011333333949247995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,3,0.01099733387430509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,131071,0.701685349146525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,7,0.011370666325092316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,15,0.011125333607196808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,63,0.011418666690587997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,127,0.011125333607196808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,31,0.011242666592200598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,255,0.014501333236694336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,511,0.019845332950353622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,63,0.15652267138163248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,4095,0.10583466291427612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1023,0.03162133445342382
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,2047,0.05729066828886668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,8191,0.20087466637293497
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,16383,0.39195199807484943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,32767,0.798858642578125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1,0.010992000500361124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,127,0.19353600343068442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,3,0.0415786678592364
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,7,0.011157333850860596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,15,0.011381333072980246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,31,0.011418666690587997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,65535,1.5397440592447917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,127,0.011450666934251785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,255,0.014384000251690546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,131071,3.0673599243164062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,511,0.02011200040578842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,255,0.3696693181991577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1023,0.03156266609827677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,2047,0.05746666590372721
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,8191,0.2026240030924479
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,16383,0.3936533530553182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1,0.010666667173306147
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,3,0.010832000523805618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,7,0.010992000500361124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,4095,0.1071519951025645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,15,0.010858666151762009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,63,0.01073066641887029
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,63,0.01166933278242747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,127,0.010687999427318573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,255,0.013967999567588171
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,511,0.019178666174411774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1023,0.030954666435718536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,31,0.010496000448862711
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,2047,0.054330666859944664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,131071,3.0903307596842446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,4095,0.10081066687901814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,8191,0.19338667392730713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,65535,1.5382827123006184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,16383,0.3791360060373942
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,511,0.6997439861297607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,32767,0.7503466606140137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,3,0.010885333021481832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,7,0.011125333607196808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,15,0.011039999624093374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,63,0.011045332998037338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,127,0.010981333752473196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1,0.0107893335322539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,511,0.01945066700379054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,255,0.013999999811251959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,131071,3.0173972447713218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,65535,1.5260000228881836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1023,0.031141333281993866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,2047,0.05444799860318502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,4095,0.10061867038408916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,8191,0.19349332650502524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,16383,0.38305068016052246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,32767,0.7635359764099121
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1,0.01110400011142095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,65535,1.5125494003295898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,32767,0.7789279619852701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,3,0.011098666737476984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,7,0.011312000453472137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,15,0.011418666690587997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,131071,3.020266532897949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,31,0.011525332927703857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,63,0.011402666568756104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,127,0.01166933278242747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,255,0.014741333822409311
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,511,0.02042666698495547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1023,0.03472000112136205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,8191,0.20964799324671426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,2047,0.06171200176080068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,31,0.011029332876205444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,16383,0.4089973370234172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,4095,0.11167466640472412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,3,0.010672000547250112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1,0.010693332801262537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,32767,0.8018346627553304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,15,0.01102399950226148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,31,0.010879999647537867
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,63,0.01099733387430509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,127,0.011114666859308878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,255,0.01404800017674764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,7,0.010911999891201654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1023,0.031189332405726116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,511,0.019573333362738293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,2047,0.05440000196297964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,4095,0.10097600022951762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,8191,0.19398399194081625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,16383,0.38764798641204834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,32767,0.7654773394266764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,65535,1.6276747385660808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,65535,1.5168479283650715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1,0.014032000054915747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,3,0.014058666924635569
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,131071,3.02782408396403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,15,0.014186666657527288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,7,0.014208000153303146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,131071,3.245493253072103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,31,0.014069333672523499
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,63,0.014485333114862442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,127,0.014831999937693277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,255,0.019098666807015736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,511,0.03225066761175791
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,8191,0.40910398960113525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1023,0.05778666834036509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,16383,0.8046613534291586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,4095,0.20210667451222739
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1,0.01834133391578992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,7,0.018624000251293182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1023,1.3541653951009114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,3,0.0183999997874101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,32767,1.5845866203308105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,2047,0.11150933305422465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,15,0.018826667219400406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,31,0.01865600049495697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,63,0.01877333347996076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,127,0.01941866676012675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,65535,3.1634934743245444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,255,0.035573333501815796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,511,0.06479466458161671
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,8191,0.8482933044433594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,2047,0.21149865786234537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,16383,1.6871253649393718
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,4095,0.40993066628774005
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,3,0.025311999022960663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,15,0.02548266698916753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1,0.025199999411900837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,7,0.025498665869235992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,31,0.025616000096003216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,255,0.055402666330337524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,63,0.025914666553338368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1,0.010847999403874079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,3,0.010741333166758219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,7,0.010938666760921478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,15,0.010746666540702185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,511,0.09955733021100362
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,31,0.010863999525705973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,127,0.01116266722480456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,255,0.013999999811251959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,511,0.019653332730134327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1023,0.11217066645622253
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1023,0.03124266614516576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,63,0.010954666882753372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,8191,0.198469340801239
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,4095,0.10130133231480916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,127,0.033530667424201965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,2047,0.05479466418425242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,16383,0.3887893358866374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,32767,0.7672639687856039
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1023,0.1909066637357076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,65535,1.5260426203409831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,8191,1.424565315246582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,2047,0.3645706574122111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,131071,3.0393174489339194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,4095,0.6760906378428141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1,0.04206933577855428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,3,0.0424586683511734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,7,0.04332800209522247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,127,0.058448001742362976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,31,0.04334400097529093
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,15,0.043141335248947144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,63,0.04924266537030538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,511,0.17600532372792563
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1023,0.33009066184361774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,255,0.10058133800824483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,2047,0.6840853691101074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1,0.07464000085989635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,4095,1.347376028696696
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,3,0.07499733567237854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,7,0.07762666543324788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,15,0.07723199824492137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1,0.011178666104873022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,63,0.08526933193206787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,3,0.011237333218256632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,7,0.01129066695769628
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,31,0.011301333705584208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,15,0.011296000331640244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,63,0.01119999960064888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,127,0.011328000575304031
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,255,0.014384000251690546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,31,0.08458133538564046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,511,0.019920000185569126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1023,0.03158933420976003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,2047,0.05487999816735586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,4095,0.10404800375302632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,8191,0.20021865765253702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,16383,0.3905653158823649
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,32767,0.7705066998799642
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,65535,1.5328586896260579
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,131071,3.0393813451131186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,255,0.1820533275604248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,127,0.10161067048708598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,511,0.33452800909678143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1023,0.6837866306304932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,2047,1.3454240163167317
