framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,3,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,7,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,15,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,31,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,63,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,255,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,2047,0.04863999783992767
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,127,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,511,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,3,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,15,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,7,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,31,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,63,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,127,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,255,0.029696000119050343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,4095,0.11895466844240825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,2047,0.0727040022611618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,7,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,15,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,63,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,127,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,255,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,511,0.05444266895453135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1023,0.07441066702206929
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,2047,0.11741866668065389
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,4095,0.07065600156784058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,4095,0.19899733861287436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,3,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,7,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,15,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,31,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,63,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,127,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1023,0.1250986655553182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,255,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,2047,0.22408533096313477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,511,0.07355733215808868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,4095,0.4203519821166992
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,255,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1023,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,511,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,2047,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,4095,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,8191,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,16383,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,7,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1023,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,2047,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,4095,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,8191,0.03566933423280716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,16383,0.040618665516376495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,2047,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,4095,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,8191,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,16383,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,255,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1023,0.04727466901143392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,2047,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,4095,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,8191,0.040448000033696495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,16383,0.06451199948787689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,15,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1023,0.051541333397229515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,511,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,3,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,255,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1023,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,8191,0.037717332442601524
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,16383,0.04266666869322459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,4095,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,2047,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,31,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,7,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,63,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,127,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1023,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,2047,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,4095,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,8191,0.03822933385769526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,16383,0.05614933371543884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,127,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,2047,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1023,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,4095,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,8191,0.048810665806134544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,16383,0.07389866809050243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,2047,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,511,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,8191,0.06621866424878438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,4095,0.04795733094215393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1023,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,16383,0.11827199657758077
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,3,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,15,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,7,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,31,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,63,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,127,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,255,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,511,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,3,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,7,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,2047,0.07406933108965556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,15,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1023,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,31,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,63,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,127,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,511,0.06365866462389629
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,255,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1023,0.08396800359090169
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,3,0.029696000119050343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,7,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,2047,0.13243732849756876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,31,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,15,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,63,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,127,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,255,0.04966400067011515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1023,0.12595199545224509
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1,0.03583999971548716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,2047,0.22664533058802286
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,3,0.03601066768169403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,511,0.0766293356815974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,7,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,15,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,63,0.037205333511034645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,127,0.048469334840774536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1023,0.19541333119074503
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,31,0.03618133316437403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,255,0.06929066777229309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,3,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,511,0.11315199732780457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,15,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,2047,0.3572053511937459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,2047,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1023,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,8191,0.03498666733503342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,4095,0.03054933249950409
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,16383,0.053247998158137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,255,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,127,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,511,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1023,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,4095,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,8191,0.05171200136343638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,16383,0.09762133161226909
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,2047,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,3,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,7,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,15,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,255,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,511,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1023,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,2047,0.03259733319282532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,4095,0.04744533201058706
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,8191,0.0679253339767456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,16383,0.11059199770291646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,15,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,63,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,127,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,255,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,511,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1023,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,2047,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,4095,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,8191,0.10734933614730835
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,16383,0.18636800845464072
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,3,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,7,0.01757866640885671
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,15,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,31,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,63,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,127,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,255,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,3,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,7,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,511,0.07031466563542683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,15,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,31,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1023,0.08721066514650981
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,63,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,127,0.03754666695992152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,255,0.057002668579419456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,511,0.08345599969228108
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1023,0.13550933202107748
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,15,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,3,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,7,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,31,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,63,0.04351999859015147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,255,0.0773119976123174
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,511,0.12219732999801636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1,0.056661332647005715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,127,0.053247998158137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,3,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,7,0.05751466751098633
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,15,0.05751466751098633
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1023,0.20462934176127115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,31,0.06007466713587443
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,63,0.06502399841944377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1023,0.3524266481399536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,127,0.07577600081761678
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,255,0.12526933352152506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,1,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,511,0.20053333044052124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,7,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,31,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,15,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,3,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,63,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,127,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,255,0.04915200173854828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,1,0.04898133377234141
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,7,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,3,0.049322664737701416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,15,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,31,0.051541333397229515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,63,0.05717333157857259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,127,0.06519466638565063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,1,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,255,0.08994133273760478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,3,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,7,0.06758399804433186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,31,0.07338666419188182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,15,0.06946133573849995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,63,0.07714133461316426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,1,0.09693866968154907
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,3,0.09830400347709656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,127,0.09028266867001851
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,255,0.13567999998728433
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,7,0.10308266679445903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,15,0.105813334385554
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,31,0.10700800021489461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,63,0.10871466994285583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,127,0.13056000073750815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,255,0.21742933988571167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,1,0.03583999971548716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,3,0.03549866626660029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,7,0.03669333209594091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,15,0.03942399968703588
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,31,0.04232533276081085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,63,0.04693333307902018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,127,0.05358933409055074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,1,0.12270933389663696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,7,0.09403733412424724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,3,0.09301333626111348
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,15,0.09506133198738098
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,31,0.09727999567985535
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,1,0.12526933352152506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,63,0.10086400310198466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,127,0.11622400085131328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,3,0.12902399897575378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,7,0.130730668703715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,15,0.13056000073750815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,31,0.13004799683888754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,63,0.13346133629480997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,1,0.18636800845464072
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,127,0.15871999661127725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,7,0.19575466712315878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,7,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,3,0.19165867567062378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,15,0.19336533546447754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,3,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1023,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,31,0.1930239995320638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,255,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,511,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,2047,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,8191,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,16383,0.0718506673971812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,4095,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,15,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,31,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,63,0.19575466712315878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,63,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,3,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,255,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,127,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,511,0.048469334840774536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1023,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,2047,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,127,0.24337067206700644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,8191,0.07406933108965556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,16383,0.11724799871444702
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,3,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,7,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,15,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,31,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,127,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,63,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,2047,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,4095,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1023,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,511,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,4095,0.04966400067011515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,16383,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,3,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,8191,0.10939733187357585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,15,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,7,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,31,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,63,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,127,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,255,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1023,0.04351999859015147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,2047,0.06741333504517873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,511,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,4095,0.10871466994285583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,8191,0.18978132804234824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,255,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,16383,0.3433813254038493
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,1,0.06980266670385997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,1,0.17271467049916586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,7,0.07014399766921997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,3,0.06963199873765309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,63,0.08038400113582611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,15,0.07236266632874806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,31,0.07441066702206929
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,3,0.17169066270192465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,7,0.17322667439778647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,1,0.23773866891860962
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,31,0.17390932639439902
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,63,0.20974934101104736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,15,0.17271467049916586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,3,0.23808000485102335
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,7,0.2409813404083252
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,15,0.23961599667867026
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,1,0.3601066668828328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,31,0.2402986685434977
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,3,0.3614720106124878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,63,0.24661332368850708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,7,0.36420265833536786
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,15,0.36369065443674725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,31,0.3679573138554891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,63,0.37444265683492023
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,1,0.1327786644299825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,3,0.13397333025932312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,1,0.32631466786066693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,7,0.13619200388590494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,15,0.13499733805656433
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,31,0.1397760013739268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,3,0.32682667175928753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,7,0.3293866713841756
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,15,0.32733867565790814
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,1,0.4541440010070801
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,7,0.46404266357421875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,31,0.3232426643371582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,3,0.4561920166015625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,15,0.4638719956080119
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,3,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,7,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,31,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,63,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,31,0.4648960034052531
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,127,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,15,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,511,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,2047,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1023,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,8191,0.06997333467006683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,4095,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,255,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,3,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,7,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,31,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,15,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,63,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,127,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,255,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,511,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,1,0.6976853211720785
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1023,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,4095,0.07082666456699371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,2047,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,8191,0.11690666278203328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,3,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,15,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,31,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,7,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,63,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,127,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,255,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,2047,0.07150933146476746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,4095,0.11212799946467082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,511,0.03259733319282532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1023,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,8191,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,3,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,7,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,15,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,31,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,63,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,127,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,255,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,511,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1023,0.07031466563542683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,2047,0.11485866705576579
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,4095,0.19165867567062378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,8191,0.3452586730321248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,3,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,7,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,15,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,3,0.7007573445638021
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,31,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,63,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,127,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,255,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,511,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1023,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,2047,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,4095,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,3,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,7,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,15,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,7,0.7080960273742676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,63,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,127,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,31,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,255,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,511,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1023,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,4095,0.11622400085131328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,2047,0.07133866846561432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,15,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,3,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,7,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,31,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,127,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,63,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,255,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,511,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,2047,0.12236799796422322
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,4095,0.1945599913597107
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,3,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,7,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,15,0.7120213508605957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,15,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,31,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1023,0.07355733215808868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,63,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,127,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,255,0.04437333345413208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1023,0.12356266379356384
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,4095,0.420693318049113
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,511,0.07372800012429555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,2047,0.22374399503072104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1023,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,2047,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,4095,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,16383,0.049322664737701416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,8191,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,3,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,15,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,63,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,127,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,2047,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,4095,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,16383,0.039936001102129616
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,8191,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,4095,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,2047,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,8191,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,16383,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1023,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,4095,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,8191,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,2047,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,16383,0.06365866462389629
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,255,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,511,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,2047,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,4095,0.02611200014750163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1023,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,8191,0.036864000062147774
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,16383,0.04181333382924398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,255,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,127,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,511,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1023,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,31,0.7181653181711832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,4095,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,8191,0.03788800040880839
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,2047,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,16383,0.05649066468079885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,63,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1023,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,2047,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,4095,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,16383,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,8191,0.04710400104522705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,127,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,255,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1023,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,511,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,2047,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,4095,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,8191,0.06621866424878438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,16383,0.10752000411351521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,511,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,3,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,15,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,7,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,63,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,31,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,127,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,511,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,255,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,3,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,7,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1023,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,31,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,15,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,2047,0.07031466563542683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,63,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,255,0.02918400118748347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,127,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,511,0.05717333157857259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,7,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,3,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,15,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,2047,0.12458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,63,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,127,0.029696000119050343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,31,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1023,0.07714133461316426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,255,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,3,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,511,0.07441066702206929
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1023,0.12441600362459819
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,2047,0.2249386707941691
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,7,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,15,0.03549866626660029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,31,0.03549866626660029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,127,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,63,0.03601066768169403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,255,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1023,0.19370667139689127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,15,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,7,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,2047,0.3577173153559367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,255,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,511,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,2047,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1023,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,8191,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,511,0.11246933539708455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,16383,0.05205333232879639
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,4095,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,15,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,511,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1023,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,2047,0.029696000119050343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,4095,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,8191,0.0506879985332489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,16383,0.07509333391984303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,31,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,63,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,255,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,511,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1023,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,2047,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,8191,0.0682666649421056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,4095,0.04863999783992767
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,16383,0.10803199807802837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,15,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,31,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,63,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,255,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,511,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1023,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,2047,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,4095,0.06604800124963124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,8191,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,16383,0.1879040002822876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,3,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,15,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,7,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,31,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,63,0.01757866640885671
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,127,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,255,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,511,0.06297599772612254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,3,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,7,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,15,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,31,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1023,0.08123733103275299
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,127,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,63,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,255,0.05358933409055074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1,0.03839999934037527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,511,0.07867733140786488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1023,0.130730668703715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,3,0.03874133278926214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,31,0.03908266623814901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,7,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,15,0.03908266623814901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,63,0.04027733455101649
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,127,0.050517335534095764
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,255,0.07321600119272868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,511,0.11724799871444702
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1,0.053930665055910744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,7,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,3,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,15,0.05529599885145823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,31,0.057002668579419456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1023,0.2039466698964437
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,63,0.06144000093142191
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,127,0.0718506673971812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1023,0.3466240167617798
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,255,0.11417599519093831
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,1,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,511,0.19524266322453818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,7,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,3,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,15,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,31,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,63,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,127,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,255,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,1,0.04437333345413208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,3,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,15,0.045567999283472695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,7,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,31,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,127,0.05922133227189382
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,1,0.06195199986298879
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,255,0.08294400076071422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,3,0.062463998794555664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,7,0.06365866462389629
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,15,0.06348800162474315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,31,0.0679253339767456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,127,0.08362666765848796
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,63,0.07389866809050243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,1,0.09267200032869975
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,3,0.09386666615804036
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,7,0.09574400385220845
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,15,0.10103467106819153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,255,0.12919466694196066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,31,0.10222933689753215
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,63,0.10427733262379964
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,255,0.2104319930076599
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,1,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,3,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,15,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,31,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,7,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,63,0.07150933146476746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,127,0.1256106694539388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,1,0.07816533247629802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,3,0.07987200220425923
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,127,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,7,0.08191999793052673
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,15,0.08345599969228108
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,31,0.08584533135096233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,63,0.09130666653315227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,1,0.11366400122642517
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,127,0.10359467069307964
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,7,0.11929600437482198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,3,0.11520000298817952
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,15,0.11929600437482198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,31,0.11929600437482198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,63,0.1230506698290507
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,1,0.17425066232681274
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,127,0.14660267035166422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,3,0.17937066157658896
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,7,0.18414932489395142
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,7,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,31,0.18278400103251138
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,15,0.18500266472498575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,255,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1023,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,63,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,2047,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,127,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,8191,0.04949333270390829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,4095,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,63,0.18568533658981323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,16383,0.07202133536338806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,7,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,15,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,3,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,255,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,511,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1023,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,8191,0.06963199873765309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,4095,0.048469334840774536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,2047,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,16383,0.1155413289864858
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,31,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,15,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,31,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,63,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,255,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,511,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1023,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,2047,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,4095,0.06894933183987935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,8191,0.1088853379090627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,3,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,16383,0.19131733973821005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,15,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,31,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,63,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,127,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,255,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,511,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1023,0.04232533276081085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,2047,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,4095,0.10956799983978271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,8191,0.18858667214711508
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,16383,0.34252798557281494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,127,0.2295466661453247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,1,0.062463998794555664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,7,0.06417066852251689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,3,0.06382933259010315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,15,0.06502399841944377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,31,0.06724266707897186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,63,0.0721919983625412
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,1,0.14916266997655234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,7,0.1513813336690267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,15,0.1518933375676473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,3,0.14984533190727234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,31,0.15121066570281982
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,63,0.15769599874814352
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,1,0.21606399615605673
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,3,0.2182826598485311
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,7,0.22016000747680664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,15,0.21862399578094482
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,1,0.3408213456471761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,31,0.21862399578094482
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,63,0.2249386707941691
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,3,0.3415040175120036
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,7,0.34781865278879803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,15,0.34406399726867676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,31,0.3438933293024699
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,63,0.35072000821431476
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,1,0.12100266416867574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,3,0.12066133817036946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,15,0.12014933427174886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,7,0.12185600399971008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,31,0.12117333213488261
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,1,0.28142933050791424
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,3,0.28433066606521606
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,7,0.2874026695887248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,15,0.2872320016225179
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,31,0.28330665826797485
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,1,0.41472001870473224
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,3,0.41523198286692303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,7,0.4210346539815267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,15,0.4225706656773885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,3,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,7,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,1,0.6580906709035238
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,63,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,31,0.4227413336435954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,127,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,255,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,511,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1023,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,4095,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,2047,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,8191,0.06946133573849995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,3,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,7,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,15,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,31,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,63,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,127,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,255,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,511,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1023,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,2047,0.049322664737701416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,4095,0.06997333467006683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,8191,0.11315199732780457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,3,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,7,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,31,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,15,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,63,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,127,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,255,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,511,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1023,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,2047,0.06980266670385997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,4095,0.11212799946467082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,8191,0.19336533546447754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,3,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,7,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,15,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,31,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,63,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,127,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,3,0.6650880177815756
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,255,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,511,0.04898133377234141
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1023,0.06911999980608623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,2047,0.11383466919263203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,4095,0.19063466787338257
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,8191,0.3457706769307454
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,3,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,7,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,15,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,31,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,63,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,127,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,255,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,511,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1023,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,7,0.6693546772003174
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,2047,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,4095,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,3,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,7,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,15,0.6707200209299723
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,63,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,31,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,127,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,15,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,511,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,4095,0.11468799908955891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,31,0.6775466601053873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,2047,0.06963199873765309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,255,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,3,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,7,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,15,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1023,0.0506879985332489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,127,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,63,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,511,0.05273599922657013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,4095,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,31,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,2047,0.116565336783727
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1023,0.0727040022611618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,7,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,3,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,15,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,31,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,63,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,127,0.029696000119050343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1023,0.12322133779525757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,2047,0.22340265909830728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,255,0.04420266548792521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,255,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,511,0.07287466526031494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,4095,0.44509867827097577
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1023,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,2047,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,4095,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,8191,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,16383,0.048810665806134544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,3,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,2047,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,4095,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,8191,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,16383,0.040106666584809623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1023,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,2047,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,4095,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,8191,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,16383,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1023,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,2047,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,8191,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,4095,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,16383,0.0628053347269694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,15,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,255,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,2047,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1023,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,8191,0.03669333209594091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,4095,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,16383,0.04164266586303711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,7,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,31,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,255,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1023,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,2047,0.02611200014750163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,4095,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,16383,0.05341866612434387
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,8191,0.037717332442601524
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,255,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,2047,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1023,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,4095,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,8191,0.04727466901143392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,16383,0.07014399766921997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,63,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,127,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,255,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,511,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,2047,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1023,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,4095,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,8191,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,16383,0.10717866818110149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,3,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,15,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,7,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,31,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,63,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,127,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,255,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,511,0.03054933249950409
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1023,0.048469334840774536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,3,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,7,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,15,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,31,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,2047,0.06775466601053874
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,255,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,63,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,127,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,511,0.05580799778302511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1023,0.07577600081761678
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,3,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,2047,0.12322133779525757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,7,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,15,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,63,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,127,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,31,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,255,0.04863999783992767
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,2047,0.2239146629969279
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,511,0.07406933108965556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,3,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1023,0.12322133779525757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,7,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,15,0.03498666733503342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,63,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,127,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,31,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,255,0.0679253339767456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1023,0.19387733936309814
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,15,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,2047,0.3563520113627116
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,511,0.11776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,511,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,255,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,2047,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1023,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,4095,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,16383,0.05085866649945577
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,8191,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,31,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,127,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,255,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1023,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,2047,0.02918400118748347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,4095,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,8191,0.05273599922657013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,16383,0.0747519979874293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,3,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,7,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,511,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1023,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,2047,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,4095,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,8191,0.0679253339767456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,16383,0.1088853379090627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,7,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,15,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,31,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,255,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1023,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,511,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,2047,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,4095,0.06621866424878438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,8191,0.10769066214561462
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,16383,0.18722132841746011
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,3,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,7,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,15,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,31,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,63,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,127,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,511,0.059903999169667564
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,255,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,7,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,3,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1023,0.07867733140786488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,15,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,63,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,31,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1,0.03737599899371465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,127,0.03259733319282532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,255,0.051541333397229515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,511,0.07816533247629802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1023,0.1295360028743744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,3,0.03754666695992152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,7,0.038058665891488395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,15,0.03822933385769526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,31,0.03788800040880839
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,63,0.03908266623814901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,127,0.048469334840774536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1,0.053077335158983864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,255,0.07082666456699371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,3,0.053247998158137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,511,0.11520000298817952
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,7,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,15,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1023,0.1976319948832194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,31,0.05580799778302511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,127,0.07014399766921997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,255,0.11229866743087769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,1,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,511,0.19268266359965006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,3,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,7,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1023,0.343722661336263
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,31,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,15,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,127,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,63,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,255,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,3,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,1,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,15,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,7,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,31,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,63,0.04693333307902018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,127,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,1,0.05973333120346069
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,3,0.060415998101234436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,7,0.06109866499900818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,255,0.07987200220425923
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,15,0.06195199986298879
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,31,0.06400000055631001
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,63,0.06963199873765309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,127,0.08157866696516673
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,1,0.09045333663622539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,255,0.12492799758911133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,3,0.09181867043177287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,7,0.0936959981918335
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,15,0.09796266754468282
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,31,0.09966933727264404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,63,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,255,0.20616533358891806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,127,0.1225386659304301
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,1,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,3,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,15,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,7,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,63,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,31,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,127,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,1,0.07202133536338806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,7,0.07372800012429555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,3,0.07338666419188182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,15,0.07679999868075053
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,31,0.0795306662718455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,63,0.0846506655216217
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,127,0.09710933764775594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,1,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,3,0.10786133011182149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,7,0.11212799946467082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,15,0.11332266529401143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,31,0.11451733112335205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,63,0.11793067057927449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,1,0.16708266735076904
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,127,0.14011733730634054
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,3,0.1728853384653727
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,7,0.17834667364756265
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,15,0.1781760056813558
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,31,0.17851734161376953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,255,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1023,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,8191,0.04744533201058706
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,2047,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,4095,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,16383,0.07133866846561432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,3,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,15,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,31,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,63,0.17971199750900269
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,63,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,127,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,2047,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1023,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,4095,0.04761599997679392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,511,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,8191,0.06860800087451935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,16383,0.11520000298817952
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,127,0.22203733523686728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,7,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,15,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,255,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,511,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1023,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,2047,0.04659200211366018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,4095,0.07236266632874806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,8191,0.1088853379090627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,16383,0.19080533583958945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,63,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,15,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,31,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,127,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,255,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,63,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,511,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1023,0.041984001795450844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,2047,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,4095,0.10820266604423523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,8191,0.1971199909845988
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,16383,0.3428693215052287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,1,0.059392000238100685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,7,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,3,0.05973333120346069
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,31,0.06468266745408376
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,15,0.061610668897628784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,63,0.06809600194295247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,1,0.13755733768145242
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,3,0.13926399747530618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,15,0.1418239971001943
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,7,0.14165332913398743
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,31,0.14148267110188803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,63,0.1474560002485911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,1,0.20462934176127115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,3,0.20770132541656494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,7,0.21128533283869425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,15,0.20770132541656494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,31,0.20889600118001303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,63,0.21435733636220297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,1,0.32767999172210693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,3,0.33160533507664997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,7,0.337066650390625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,15,0.3327999909718831
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,31,0.33399466673533124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,63,0.34508800506591797
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,1,0.11212799946467082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,3,0.1129813293615977
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,7,0.11485866705576579
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,15,0.11434666315714519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,31,0.11332266529401143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,1,0.2616320053736369
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,3,0.2621440092722575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,7,0.2662400007247925
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,15,0.2667520046234131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,1,0.39509332180023193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,31,0.2629973292350769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,3,0.3952639897664388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,7,0.3997013171513875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,15,0.4007253249486287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,1,0.6398293177286783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,31,0.40396801630655926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,7,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,3,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,15,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,31,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,7,0.6505813201268514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,3,0.6410239934921265
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,63,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1023,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,511,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,8191,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,255,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,2047,0.030378667016824085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,4095,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,3,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,7,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,15,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,31,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,63,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,255,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,127,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1023,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,511,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,2047,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,4095,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,8191,0.11468799908955891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,3,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,7,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,15,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,31,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,63,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,127,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,255,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,511,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1023,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,2047,0.06911999980608623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,4095,0.11127466956774394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,8191,0.19404800732930502
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,3,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,15,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,7,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,31,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,63,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,127,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,255,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1023,0.07150933146476746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,15,0.649727980295817
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,511,0.04898133377234141
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,4095,0.18943999210993448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,2047,0.11434666315714519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,7,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,15,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,8191,0.3454293409983317
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,127,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,255,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,511,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1023,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,8191,0.11485866705576579
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,2047,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,4095,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,7,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,3,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,15,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,127,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,31,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,511,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,255,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,63,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1023,0.04710400104522705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,2047,0.07065600156784058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,4095,0.11315199732780457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,8191,0.19319466749827066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,31,0.6568959951400757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,3,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,15,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,31,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,63,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,7,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,255,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,2047,0.1153706709543864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,511,0.05034666756788889
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,4095,0.19217065970102945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1023,0.0718506673971812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,8191,0.3490133285522461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,7,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,3,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,31,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,15,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,63,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,127,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,127,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1023,0.12424533565839131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,511,0.07287466526031494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,2047,0.22323199113210043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,255,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,4095,0.41881601015726727
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,7,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,127,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1023,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,8191,0.8192000389099121
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,4095,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,2047,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,8191,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,16383,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,32767,0.053247998158137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,511,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,8191,0.03618133316437403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,2047,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1023,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,4095,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,16383,0.04027733455101649
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,32767,0.05563733478387197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,2047,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,4095,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,8191,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,16383,0.04693333307902018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,32767,0.07099733253320058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1023,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,2047,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,4095,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,8191,0.040618665516376495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,16383,0.06331733365853627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,32767,0.106495996316274
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,2047,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1023,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,8191,0.03618133316437403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,4095,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,16383,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,32767,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,15,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,7,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,31,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,63,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1023,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,511,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,2047,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,4095,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,8191,0.03857066730658213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,16383,0.05444266895453135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,32767,0.07816533247629802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,127,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,31,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,511,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,2047,0.0290133332212766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1023,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,4095,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,8191,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,16383,0.07116800049940745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,32767,0.11178666353225708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,127,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1023,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,2047,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,4095,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,8191,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,16383,0.10700800021489461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,32767,0.18875734011332193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,3,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,7,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,15,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,31,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,63,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,127,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,255,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,511,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1023,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,3,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,2047,0.06690133114655812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,7,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,15,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,4095,0.10922666390736897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,31,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,63,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,127,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,255,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,511,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1023,0.0747519979874293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,4095,0.19660800695419312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,2047,0.12134400010108948
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,15,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,31,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,3,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,7,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,127,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,255,0.04693333307902018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,2047,0.22528000672658285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,511,0.0727040022611618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,4095,0.4208639860153198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,63,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,3,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,15,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,31,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,63,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,7,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1023,0.1295360028743744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,127,0.04454400142033895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1023,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,2047,0.35498666763305664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,255,0.06690133114655812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,511,0.1114453375339508
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,511,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1023,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,2047,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,8191,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,16383,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,32767,0.07287466526031494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,3,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,4095,0.681984027226766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,127,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1023,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,4095,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,8191,0.048810665806134544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,16383,0.07150933146476746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,2047,0.02918400118748347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,4095,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,32767,0.11673600474993388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,511,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,2047,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,4095,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,16383,0.10922666390736897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,8191,0.06741333504517873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1023,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,32767,0.19165867567062378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,15,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,31,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,255,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,127,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,511,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,4095,0.06553600231806438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,2047,0.04266666869322459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,8191,0.10683733224868774
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,16383,0.18602667252222696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,32767,0.33894399801890057
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,3,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,7,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,15,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,31,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,127,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,255,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,511,0.05751466751098633
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1023,0.07611733178297679
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,3,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,2047,0.12270933389663696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,7,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,15,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,31,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,63,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,127,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,511,0.07679999868075053
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,255,0.04966400067011515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1023,0.12748799721399942
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,3,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,7,0.03703466554482778
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,15,0.03669333209594091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,2047,0.22835199038187662
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,31,0.03703466554482778
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,63,0.037717332442601524
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1023,0.1962666710217794
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,127,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,255,0.06980266670385997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,2047,0.362496018409729
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,511,0.11400533715883891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1,0.05239466826121012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,7,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,31,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,15,0.05273599922657013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,63,0.017407999684413273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,63,0.05870933334032694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,127,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1023,0.3415040175120036
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,3,0.05222400029500326
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,255,0.10973866780598958
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,511,0.20206934213638306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,2047,0.7005866368611654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,1,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,15,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,7,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,3,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,31,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,255,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,1,0.04095999896526337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,63,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,511,0.06877866884072621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,3,0.04130133241415024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,127,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,7,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,15,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,31,0.04181333382924398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,63,0.04369066655635834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,127,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,255,0.077824001510938
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,1,0.05853866537412008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,511,0.12782933314641318
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,3,0.05853866537412008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,7,0.05973333120346069
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,15,0.0602453351020813
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,31,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,63,0.06758399804433186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,1,0.08874666690826416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,255,0.12714667121569315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,127,0.07901866734027863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,3,0.08994133273760478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,7,0.091648002465566
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,15,0.09523199995358785
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,511,0.2034346659978231
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,31,0.09745066364606221
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,127,0.11707733074824016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,63,0.09949866930643718
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,255,0.2032639980316162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,1,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,7,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,511,0.35839998722076416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,3,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,15,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,31,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,63,0.036864000062147774
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,127,0.04625066618124644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,3,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,255,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,1,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,7,0.06741333504517873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,15,0.07014399766921997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,31,0.07389866809050243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,63,0.07799466451009114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,127,0.09096533060073853
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,1,0.09915733337402344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,255,0.13755733768145242
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,7,0.10496000448862712
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,3,0.10069333513577779
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,15,0.10820266604423523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,31,0.10956799983978271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,63,0.11212799946467082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,127,0.13482667009035745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,255,0.21964800357818604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,1,0.1604266663392385
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,3,0.16810667514801025
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,7,0.1730560064315796
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,15,0.17390932639439902
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,31,0.17203199863433838
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,15,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,63,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,255,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,511,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1023,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,2047,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,8191,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,16383,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,4095,0.03054933249950409
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,32767,0.1186133325099945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,63,0.17544533809026083
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,7,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,31,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,63,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,127,0.21623466412226358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,511,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1023,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,2047,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,8191,0.0682666649421056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,16383,0.11332266529401143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,32767,0.1930239995320638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,7,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,15,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,127,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,31,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,63,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,255,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,2047,0.044031997521718345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,511,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1023,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,4095,0.06929066777229309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,8191,0.10666666428248088
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,4095,0.0506879985332489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,16383,0.1909760038057963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,15,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,32767,0.3471359809239705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,31,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,63,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,127,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,255,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,511,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1023,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,2047,0.06604800124963124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,4095,0.10854400197664897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,8191,0.1868799924850464
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,16383,0.3413333495457967
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,32767,0.6464853286743164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,255,0.3752959966659546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,1,0.054101333022117615
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,3,0.05529599885145823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,7,0.05614933371543884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,15,0.05836800237496694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,31,0.06075733403364817
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,63,0.06570666531721751
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,127,0.08277333279450734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,1,0.12663466731707254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,3,0.12919466694196066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,7,0.13090133666992188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,15,0.13124266266822815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,63,0.1346560021241506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,31,0.13090133666992188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,127,0.16571733355522156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,1,0.19319466749827066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,3,0.1971199909845988
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,7,0.20036266247431436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,15,0.19848533471425375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,31,0.19831466674804688
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,63,0.20411733786265054
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,1,0.31624533732732135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,3,0.3213653365770976
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,127,0.24951465924580893
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,15,0.3242666721343994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,7,0.32716800769170123
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,31,0.3222186764081319
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,63,0.3304106593132019
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,127,0.41608532269795734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,1,0.10410666465759277
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,7,0.10717866818110149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,15,0.10564266641934712
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,3,0.10444800059000652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,31,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,63,0.11178666353225708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,3,0.24132267634073892
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,1,0.24012800057729086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,7,0.24473599592844644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,15,0.24541866779327393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,31,0.24371200799942017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,1,0.37614933649698895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,63,0.2476373314857483
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,7,0.3810986677805583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,15,0.40123732884724933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,31,0.4065279960632324
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,63,0.3891199827194214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,7,0.6314666668574015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,3,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,3,0.37512532869974774
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,7,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,31,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,63,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,255,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,511,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1023,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,2047,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,4095,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,8191,0.06570666531721751
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,16383,0.11059199770291646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,15,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,31,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,63,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,127,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,255,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,511,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1023,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,2047,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,4095,0.06894933183987935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,8191,0.11110400160153706
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,16383,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,3,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,7,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,15,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,31,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,3,0.6256639957427979
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,63,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,127,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,255,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,1,0.6183253526687622
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,511,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,2047,0.06877866884072621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1023,0.04369066655635834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,8191,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,4095,0.11776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,16383,0.34696535269419354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,3,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,7,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,31,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,63,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,15,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,127,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,255,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1023,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,511,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,2047,0.11827199657758077
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,4095,0.187391996383667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,8191,0.34201598167419434
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,3,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,7,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,16383,0.6481920083363851
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,31,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,63,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,127,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,255,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1023,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,511,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,8191,0.10786133011182149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,2047,0.04437333345413208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,3,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,7,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,4095,0.06519466638565063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,15,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,31,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,63,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,255,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,511,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,127,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1023,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,4095,0.1129813293615977
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,8191,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,2047,0.06929066777229309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,3,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,15,0.629589319229126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,7,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,15,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,31,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,63,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,127,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,255,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,2047,0.11246933539708455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1023,0.07099733253320058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,4095,0.1914880077044169
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,511,0.051370665431022644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,8191,0.34833065668741864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,31,0.634879986445109
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,7,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,3,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,31,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,15,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,63,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,127,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1023,0.12288000186284383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,2047,0.2230613430341085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,63,0.6411946614583334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,4095,0.4184746742248535
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,127,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,511,0.0721919983625412
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1023,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,2047,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,4095,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,16383,0.04761599997679392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,3,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,32767,0.05239466826121012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,7,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,255,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,127,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,8191,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,511,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1023,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,4095,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,2047,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,8191,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,16383,0.03942399968703588
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,32767,0.05580799778302511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,7,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,8191,0.8668159643809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,255,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,8191,0.03310933212439219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,2047,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,16383,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,32767,0.06911999980608623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,4095,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,2047,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,8191,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,16383,0.06365866462389629
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,32767,0.10734933614730835
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,4095,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,127,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,2047,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,8191,0.03566933423280716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,16383,0.039936001102129616
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,4095,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,32767,0.06195199986298879
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,7,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,63,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,127,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,255,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,511,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1023,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,4095,0.03259733319282532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,8191,0.037205333511034645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,2047,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,16383,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,32767,0.07560533285140991
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,63,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,127,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,511,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1023,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,2047,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,4095,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,8191,0.04642133414745331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,16383,0.06877866884072621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,32767,0.1104213297367096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,3,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,7,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,31,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,63,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,255,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,2047,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1023,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,4095,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,8191,0.06570666531721751
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,16383,0.10752000411351521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,32767,0.1867093245188395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,3,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,7,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,15,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,31,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,63,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,127,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,255,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,3,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,511,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1023,0.04505600035190582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,7,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,15,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,31,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,2047,0.0653653343518575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,63,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,255,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,127,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,4095,0.1083733340104421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1023,0.0718506673971812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,3,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,511,0.053077335158983864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,7,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,4095,0.1962666710217794
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,31,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,2047,0.11776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,127,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,255,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,63,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,2047,0.22340265909830728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,511,0.07355733215808868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,15,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1023,0.12288000186284383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,7,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,3,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,4095,0.42018131415049237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,31,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,63,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,15,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,127,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1023,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,255,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,2047,0.35447466373443604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,31,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,4095,0.6814719835917155
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,511,0.11588266491889954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1023,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,4095,0.029696000119050343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,2047,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,16383,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,8191,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,32767,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,127,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,511,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1023,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,4095,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,2047,0.02918400118748347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,8191,0.04795733094215393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,16383,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,32767,0.12185600399971008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,7,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,511,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,2047,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1023,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,4095,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,8191,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,16383,0.1083733340104421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,32767,0.1950719952583313
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,63,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,255,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1023,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,2047,0.041984001795450844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,4095,0.06570666531721751
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,8191,0.10564266641934712
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,511,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,16383,0.1858560045560201
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,32767,0.3403093417485555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,7,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,3,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,15,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,63,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,127,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,255,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,3,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,7,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,511,0.05120000243186951
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,15,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,2047,0.11622400085131328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1023,0.07014399766921997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,31,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,63,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,127,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,255,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,511,0.07458133498827617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,7,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,3,0.03498666733503342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,15,0.03583999971548716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1023,0.1269760032494863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,2047,0.22630399465560913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,31,0.03549866626660029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,63,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,127,0.045567999283472695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,255,0.06809600194295247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1,0.0506879985332489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,3,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,511,0.11195733149846394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1023,0.1950719952583313
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,7,0.051882664362589516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,2047,0.35788798332214355
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,15,0.05171200136343638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,31,0.05341866612434387
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,63,0.0580266664425532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,127,0.06963199873765309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1023,0.3403093417485555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,255,0.10854400197664897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,2047,0.6434133450190226
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,511,0.20053333044052124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,1,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,3,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,15,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,63,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,7,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,31,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,255,0.04181333382924398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,127,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,511,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,1,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,7,0.03874133278926214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,3,0.03857066730658213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,15,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,31,0.03874133278926214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,63,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,127,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,255,0.07321600119272868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,1,0.05597866574923197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,511,0.11895466844240825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,7,0.056661332647005715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,3,0.05649066468079885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,15,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,31,0.05870933334032694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,63,0.06451199948787689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,127,0.07338666419188182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,1,0.08618666728337605
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,15,0.09079466263453166
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,7,0.08891733487447102
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,255,0.11776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,3,0.08772266904513042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,511,0.19643733898798624
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,31,0.09437867005666097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,63,0.09540266791979472
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,127,0.11212799946467082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,255,0.1976319948832194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,511,0.3510613441467285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,1,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,3,0.0290133332212766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,15,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,7,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,31,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,63,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,127,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,1,0.061610668897628784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,3,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,7,0.0628053347269694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,255,0.06775466601053874
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,15,0.0631466656923294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,31,0.06758399804433186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,63,0.0727040022611618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,1,0.09540266791979472
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,127,0.08533333738644917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,3,0.0962559978167216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,255,0.13090133666992188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,7,0.0981333355108897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,15,0.1032533347606659
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,31,0.10393599669138591
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,63,0.106495996316274
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,1,0.1551359991232554
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,127,0.1539413332939148
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,255,0.2135039965311686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,3,0.1604266663392385
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,7,0.16708266735076904
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,15,0.16844799121220908
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,31,0.16639999548594156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,255,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,511,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1023,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,63,0.16861865917841592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,2047,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,4095,0.030378667016824085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,8191,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,3,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,32767,0.11246933539708455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,15,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,31,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,63,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,127,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,511,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,16383,0.06980266670385997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1023,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,4095,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,2047,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,8191,0.06860800087451935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,16383,0.11895466844240825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,32767,0.19490132729212442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,15,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,63,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,255,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,511,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1023,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,2047,0.044031997521718345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,4095,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,8191,0.10666666428248088
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,16383,0.1879040002822876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,127,0.20753065745035806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,32767,0.3444053332010905
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,63,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,127,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1023,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,255,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,2047,0.06587733328342438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,4095,0.10803199807802837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,8191,0.1853440006573995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,16383,0.340992013613383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,32767,0.6451199849446615
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,255,0.3882666826248169
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,511,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,1,0.048469334840774536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,3,0.04949333270390829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,7,0.050517335534095764
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,15,0.053077335158983864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,31,0.05717333157857259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,127,0.07492266595363617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,1,0.11229866743087769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,63,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,3,0.11571199695269267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,7,0.12032000223795573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,15,0.12014933427174886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,31,0.12117333213488261
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,1,0.18107734123865762
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,63,0.12458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,127,0.15223466356595358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,3,0.18551466862360635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,7,0.18943999210993448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,15,0.18858667214711508
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,31,0.18875734011332193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,63,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,1,0.3022506634394328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,3,0.31112533807754517
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,127,0.2469546596209208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,7,0.3188053369522095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,15,0.31470932563145954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,31,0.31334400177001953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,63,0.3198293248812358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,127,0.3990186850229899
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,1,0.0981333355108897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,3,0.09915733337402344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,7,0.09966933727264404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,15,0.10103467106819153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,31,0.09983999530474345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,63,0.10461866855621338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,1,0.2198186715443929
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,3,0.2208426594734192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,7,0.22357332706451416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,15,0.22340265909830728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,31,0.22220800320307413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,63,0.22613332668940225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,1,0.3536213239034017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,3,0.355840007464091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,7,0.3604480028152466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,15,0.358570655186971
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,1,0.5956266721089681
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,31,0.36113067468007404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,7,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,3,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,63,0.3662506739298503
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,255,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1023,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,3,0.6036479870478312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,2047,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,511,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,4095,0.04369066655635834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,8191,0.06451199948787689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,16383,0.11434666315714519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,7,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,15,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,31,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,63,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,127,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,255,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,511,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1023,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,4095,0.0682666649421056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,8191,0.11229866743087769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,2047,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,16383,0.1909760038057963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,7,0.6106453339258829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,3,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,15,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,15,0.6106453339258829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,31,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,127,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,255,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,63,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,2047,0.0682666649421056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,511,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1023,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,8191,0.20258132616678873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,16383,0.3485013246536255
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,3,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,4095,0.11827199657758077
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,7,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,15,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,31,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,63,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,127,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1023,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,2047,0.11195733149846394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,511,0.04710400104522705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,255,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,4095,0.18807466824849448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,8191,0.34167468547821045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,31,0.6155946652094523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,7,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,15,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,16383,0.6444373528162638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,63,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,511,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1023,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,8191,0.105813334385554
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,16383,0.19797333081563315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,63,0.6195199886957804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,2047,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,4095,0.06553600231806438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,3,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,15,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,255,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,63,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,127,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,511,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,31,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1023,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,255,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,4095,0.11229866743087769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,8191,0.1909760038057963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,3,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,16383,0.34935466448465985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,15,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,31,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,7,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,63,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,127,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,255,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,2047,0.1129813293615977
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,511,0.048469334840774536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,4095,0.19080533583958945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,8191,0.34645334879557294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,3,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1023,0.06963199873765309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,15,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,7,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,16383,0.6589440107345581
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,31,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,63,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,127,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1023,0.12185600399971008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,255,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,2047,0.06741333504517873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,2047,0.22272000710169473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,4095,0.41813333829243976
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,511,0.07133866846561432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,7,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,255,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,8191,0.8171520233154297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1023,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,2047,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,4095,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,8191,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,16383,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,32767,0.05358933409055074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,65535,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,16383,1.7225386301676433
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,511,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1023,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,255,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,4095,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,16383,0.03925333420435587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,2047,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,8191,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,32767,0.05597866574923197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,65535,0.07970133423805237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,15,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,31,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,255,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1023,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,4095,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,16383,0.04625066618124644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,8191,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,32767,0.06894933183987935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,2047,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,65535,0.10922666390736897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1023,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,2047,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,4095,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,16383,0.062463998794555664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,8191,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,32767,0.10513066252072652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,7,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,65535,0.18653867642084757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,127,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,8191,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1023,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,2047,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,4095,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,16383,0.040106666584809623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,32767,0.059392000238100685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,3,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,65535,0.08294400076071422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,15,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,31,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,127,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,255,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1023,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,511,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,4095,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,2047,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,8191,0.03703466554482778
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,16383,0.0554666668176651
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,32767,0.07867733140786488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,65535,0.12595199545224509
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,7,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,511,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1023,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,2047,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,4095,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,8191,0.04625066618124644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,16383,0.0727040022611618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,32767,0.1109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,7,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,31,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,65535,0.20889600118001303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1023,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,2047,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,8191,0.0653653343518575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,4095,0.04625066618124644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,16383,0.10803199807802837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,32767,0.1884160041809082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,65535,0.3717120091120402
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,3,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,7,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,15,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,31,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,63,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,127,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,255,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,511,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,8191,0.19729065895080566
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1023,0.044031997521718345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,3,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,7,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,15,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,2047,0.06604800124963124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,31,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,63,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,127,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,255,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,4095,0.106495996316274
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1023,0.07406933108965556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,511,0.0506879985332489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,4095,0.19114667177200317
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,8191,0.3553280035654704
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,2047,0.11741866668065389
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,15,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,31,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,3,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,7,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,63,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,255,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,127,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,2047,0.22425599892934164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,4095,0.4222293297449748
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,511,0.0718506673971812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,3,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1023,0.12117333213488261
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,15,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,31,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,7,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,63,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,127,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,255,0.06690133114655812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,511,0.10854400197664897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1023,0.1919999917348226
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,2047,0.3536213239034017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,4095,0.6826666990915934
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,8191,0.8690346876780192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1023,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,2047,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,4095,0.02918400118748347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,16383,0.050517335534095764
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,32767,0.06946133573849995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,65535,0.11946666240692139
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,3,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,8191,1.4424746831258137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,8191,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,511,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,2047,0.0290133332212766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,4095,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1023,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,8191,0.050517335534095764
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,16383,0.07355733215808868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,32767,0.11622400085131328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,65535,0.21248000860214233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,31,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,2047,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,511,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,4095,0.04505600035190582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,8191,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1023,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,16383,0.10854400197664897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,32767,0.19165867567062378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,65535,0.343722661336263
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,127,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,255,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,511,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1023,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,4095,0.06502399841944377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,8191,0.10666666428248088
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,2047,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,16383,0.18551466862360635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,32767,0.33740798632303876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,65535,0.6475093364715576
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,3,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,7,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,15,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,31,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,63,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,127,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,255,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,511,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,3,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1023,0.06758399804433186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,7,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,2047,0.11366400122642517
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,15,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,31,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,63,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,4095,0.19114667177200317
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,127,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1023,0.12595199545224509
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,255,0.04642133414745331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,511,0.07372800012429555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,4095,0.42393600940704346
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,7,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,2047,0.22698666652043661
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,31,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,127,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,63,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,255,0.06690133114655812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,15,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,3,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1023,0.19387733936309814
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,2047,0.355840007464091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,511,0.11195733149846394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,7,0.0506879985332489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,3,0.0506879985332489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,15,0.05085866649945577
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,4095,0.7350613276163737
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,63,0.05819733440876007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,127,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1023,0.33774932225545246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,31,0.051882664362589516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,2047,0.6379520098368326
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,255,0.10734933614730835
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,4095,1.357653299967448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,511,0.1976319948832194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,3,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,7,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,15,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,31,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,63,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,255,0.040106666584809623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,511,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,3,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,127,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,7,0.03703466554482778
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1023,0.11827199657758077
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,15,0.03737599899371465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,31,0.03703466554482778
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,63,0.03737599899371465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,127,0.04710400104522705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,255,0.06980266670385997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,511,0.1153706709543864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,3,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1,0.053930665055910744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,7,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1023,0.19694934288660684
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,15,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,31,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,63,0.061610668897628784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,127,0.07065600156784058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,3,0.08516266942024231
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,7,0.0865280032157898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,15,0.0885759989420573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,255,0.11246933539708455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,31,0.09181867043177287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1,0.08482133348782857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,511,0.19438934326171875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,63,0.09301333626111348
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,127,0.1088853379090627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1023,0.3474773168563843
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1023,0.646997332572937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,255,0.20480000972747803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,511,0.37307735284169513
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,1,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,7,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,3,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,15,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,63,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,127,0.04164266586303711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,1,0.05819733440876007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,511,0.10820266604423523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,3,0.05836800237496694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,31,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,7,0.05922133227189382
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,255,0.062122667829195656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,15,0.059392000238100685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,31,0.06144000093142191
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,63,0.06809600194295247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,127,0.08038400113582611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,1,0.09181867043177287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,255,0.1256106694539388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,511,0.2039466698964437
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,3,0.09233066439628601
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,7,0.09386666615804036
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,15,0.0962559978167216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,31,0.10001066327095032
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,63,0.10257066289583842
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,127,0.12236799796422322
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,255,0.2063360015551249
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,1,0.15172266960144043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,3,0.15291733543078104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,511,0.3604480028152466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,7,0.16076800227165222
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,15,0.16622933745384216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,31,0.16401066382726034
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,63,0.16452266772588095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,511,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1023,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,8191,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,127,0.21333332856496176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,4095,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,16383,0.06553600231806438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,32767,0.11059199770291646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,65535,0.1925119956334432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,2047,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,127,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,511,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1023,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,2047,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,4095,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,8191,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,16383,0.11246933539708455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,32767,0.19541333119074503
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,65535,0.3556693394978841
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,7,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,15,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,31,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,127,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,2047,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,511,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,255,0.35788798332214355
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,4095,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,8191,0.10752000411351521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,16383,0.1884160041809082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,7,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,32767,0.3720533450444539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,15,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,31,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,63,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,255,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,511,0.030378667016824085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,2047,0.06519466638565063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1023,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,4095,0.1153706709543864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,8191,0.18653867642084757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,16383,0.34167468547821045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,32767,0.64955735206604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,65535,0.6533120075861613
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,65535,1.3791573842366536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,511,0.6625279982884725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,1,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,3,0.04727466901143392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,7,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,15,0.049322664737701416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,31,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,63,0.05870933334032694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,255,0.11281067132949829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,127,0.07082666456699371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,1,0.09830400347709656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,7,0.10461866855621338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,15,0.10820266604423523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,31,0.11127466956774394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,63,0.11434666315714519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,127,0.1430186629295349
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,255,0.21896533171335855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,1,0.16366933782895407
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,3,0.10052266716957092
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,3,0.16964266697565714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,7,0.17783466974894205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,15,0.1781760056813558
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,31,0.17783466974894205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,63,0.18449066082636514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,127,0.22101332743962607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,1,0.2892799973487854
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,3,0.2998613317807515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,255,0.3826346794764201
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,7,0.30856533845265705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,15,0.304639995098114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,31,0.30395734310150146
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,63,0.3128319978713989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,127,0.3821226755777995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,255,0.688810666402181
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,1,0.09011200070381165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,7,0.09642666578292847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,15,0.09642666578292847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,3,0.09062400460243225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,31,0.09454933802286784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,63,0.09762133161226909
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,127,0.130730668703715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,1,0.19575466712315878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,3,0.19848533471425375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,7,0.2027519941329956
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,15,0.20258132616678873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,31,0.20155733823776245
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,63,0.2058239976565043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,127,0.25668267409006756
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,1,0.3304106593132019
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,3,0.3357013463973999
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,7,0.358570655186971
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,15,0.33740798632303876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,31,0.3397973378499349
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,63,0.3433813254038493
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,127,0.4264959891637166
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,1,0.5758293469746908
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,3,0.5835093259811401
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,255,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,511,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,7,0.5908480087916056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1023,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,2047,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,4095,0.04095999896526337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,16383,0.10717866818110149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,8191,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,32767,0.18807466824849448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,7,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,15,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,31,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,63,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,511,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1023,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,2047,0.045567999283472695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,4095,0.06690133114655812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,15,0.5894826650619507
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,8191,0.11178666353225708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,16383,0.18995199600855509
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,32767,0.3515733480453491
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,7,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,15,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,63,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,3,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,127,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,255,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,511,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1023,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,2047,0.07014399766921997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,4095,0.10922666390736897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,8191,0.187391996383667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,16383,0.3720533450444539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,3,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,7,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,31,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,15,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,32767,0.6555306514104208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,31,0.5935786565144857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,63,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1023,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,255,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,127,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,2047,0.11178666353225708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,511,0.04659200211366018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,8191,0.3433813254038493
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,16383,0.6434133450190226
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,7,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,31,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,4095,0.1879040002822876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,32767,1.369599978129069
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,255,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1023,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,8191,0.11229866743087769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,16383,0.18466132879257202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,3,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,2047,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,15,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,31,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,4095,0.06331733365853627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,63,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,127,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,255,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,2047,0.06894933183987935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1023,0.04505600035190582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,511,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,4095,0.11264000336329143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,8191,0.19182932376861572
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,16383,0.35089067618052167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,63,0.5963093439737955
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,7,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,15,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,3,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,31,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,63,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,255,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,127,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,2047,0.11076266566912334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,511,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1023,0.07082666456699371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,8191,0.34679468472798664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,3,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,15,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,255,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,511,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1023,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,16383,0.6592853466669718
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,2047,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,4095,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,8191,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,16383,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,32767,0.05205333232879639
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,65535,0.06570666531721751
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1023,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,2047,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,8191,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,4095,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,16383,0.03942399968703588
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,32767,0.05649066468079885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,4095,0.1890986760457357
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,65535,0.08038400113582611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,7,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,255,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1023,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,2047,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,127,0.7430826822916666
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,4095,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,8191,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,16383,0.04539733131726583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,32767,0.06929066777229309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,65535,0.10769066214561462
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,3,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1023,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,2047,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,4095,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,8191,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,16383,0.03976533313592275
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,32767,0.055125330885251365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,65535,0.0795306662718455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,7,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,3,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,31,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,63,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,255,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,127,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,511,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1023,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,2047,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,4095,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,8191,0.03601066768169403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,16383,0.049322664737701416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,32767,0.07423999905586243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,3,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,65535,0.11827199657758077
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,7,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,15,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,31,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,63,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,127,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,511,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1023,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,2047,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,4095,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,8191,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,16383,0.0682666649421056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,32767,0.10990933577219646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,65535,0.19217065970102945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,3,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,7,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,31,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,511,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,255,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,127,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1023,0.03874133278926214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,2047,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,511,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,8191,0.1838080088297526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,3,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,4095,0.10683733224868774
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,31,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,7,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,15,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,63,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,127,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,255,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,4095,0.19165867567062378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1023,0.07150933146476746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,511,0.05017599960168203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,8191,0.3548159996668498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,3,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,2047,0.11690666278203328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,7,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,15,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,127,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,63,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,31,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,255,0.044031997521718345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,2047,0.22323199113210043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,511,0.07287466526031494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,4095,0.42018131415049237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1023,0.12032000223795573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,8191,0.8704000314076742
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,2047,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,8191,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,4095,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1023,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,16383,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,65535,0.11349333326021831
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,7,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,3,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,63,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,31,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,127,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,511,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1023,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,4095,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,8191,0.05017599960168203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,2047,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,16383,0.06980266670385997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,32767,0.07099733253320058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,32767,0.1129813293615977
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,3,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,65535,0.20070399840672812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,63,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,31,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,15,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,511,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1023,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,2047,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,4095,0.04420266548792521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,8191,0.06570666531721751
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,16383,0.11502933502197266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,32767,0.18961066007614136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,65535,0.3447466691335042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,3,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,7,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,15,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,31,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,63,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,127,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,255,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,511,0.04693333307902018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,3,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,7,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1023,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,15,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,2047,0.11195733149846394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,4095,0.1884160041809082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,31,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,63,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,127,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,255,0.04625066618124644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,511,0.07321600119272868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1023,0.12390399972597758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,2047,0.22510933876037598
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,4095,0.4237653414408366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,3,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,7,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,15,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,63,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,255,0.06690133114655812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,31,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,2047,0.35652267932891846
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,127,0.04351999859015147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,511,0.11571199695269267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,4095,0.7430826822916666
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1023,0.2063360015551249
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,3,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,7,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,15,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,63,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,31,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,127,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,255,0.03874133278926214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,511,0.06553600231806438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,3,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,7,0.03549866626660029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,15,0.03566933423280716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1023,0.1220266620318095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,31,0.03601066768169403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,63,0.03857066730658213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,127,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,255,0.06911999980608623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,511,0.1155413289864858
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1,0.05239466826121012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,3,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1023,0.1955839991569519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,7,0.05376000205675761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,15,0.05376000205675761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,31,0.05358933409055074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,63,0.06109866499900818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,127,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,255,0.11127466956774394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,511,0.2034346659978231
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,1,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1023,0.37034666538238525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,3,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,7,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,15,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,31,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,63,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,1,0.05580799778302511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,255,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,127,0.039594667653242745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,3,0.05580799778302511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,7,0.056320001681645714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,511,0.10564266641934712
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,15,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,31,0.05734399954477946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,63,0.06417066852251689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,127,0.07594666878382365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,255,0.1204906702041626
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,1,0.08840533097585042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,7,0.09147733449935913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,511,0.1976319948832194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,3,0.08959999680519104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,15,0.09250133236249287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,255,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,31,0.09676800171534221
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,63,0.09898666540781657
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1023,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,4095,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,127,0.11741866668065389
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,2047,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,8191,0.041984001795450844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,32767,0.10990933577219646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,65535,0.19131733973821005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,16383,0.06877866884072621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,31,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,511,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,4095,0.044031997521718345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1023,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,2047,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,8191,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,16383,0.11161599556605022
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,32767,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,65535,0.3510613441467285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,31,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,511,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,255,0.20087466637293497
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,2047,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,4095,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,8191,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,16383,0.18943999210993448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,32767,0.34065067768096924
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,511,0.35601067543029785
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,65535,0.6599680185317993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,1,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,7,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,3,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,15,0.04710400104522705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,31,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,63,0.056661332647005715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,127,0.06690133114655812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,255,0.11076266566912334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,1,0.09454933802286784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,3,0.09506133198738098
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,15,0.10257066289583842
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,7,0.09796266754468282
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,31,0.10478933652242024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,63,0.10871466994285583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,127,0.13209600249926248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,255,0.21760000785191855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,1,0.15940266847610474
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,3,0.1628159979979197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,7,0.1713493267695109
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,15,0.1718613306681315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,31,0.17254400253295898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,63,0.17749333381652832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,127,0.2136746644973755
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,255,0.39560532569885254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,1,0.08328533172607422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,3,0.08601599931716919
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,7,0.08721066514650981
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,15,0.09335466225941975
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,63,0.0962559978167216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,31,0.09215999643007915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,127,0.12219732999801636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,1,0.18056533734003702
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,3,0.187391996383667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,7,0.19234132766723633
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,15,0.19114667177200317
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,31,0.19114667177200317
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,63,0.1955839991569519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,1,0.31778132915496826
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,127,0.24234666426976523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,31,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,15,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,255,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1023,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,2047,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,4095,0.03976533313592275
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,8191,0.06297599772612254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,3,0.3275093237559001
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,16383,0.10752000411351521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,32767,0.18653867642084757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,3,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,15,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,63,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,511,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1023,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,4095,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,2047,0.04420266548792521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,8191,0.11076266566912334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,16383,0.18926932414372763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,3,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,32767,0.34986666838328045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,7,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,15,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,31,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,63,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,127,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,255,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,511,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1023,0.04232533276081085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,2047,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,4095,0.1088853379090627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,8191,0.18858667214711508
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,16383,0.3421866496404012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,7,0.33160533507664997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,32767,0.7154346307118734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,3,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,7,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,127,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,255,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,511,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1023,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,2047,0.037717332442601524
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,15,0.3285333315531413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,4095,0.06178133189678192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,8191,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,16383,0.18312533696492514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,31,0.3300693432490031
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,15,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,32767,0.3432106574376424
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,7,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,31,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,3,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,127,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,255,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1023,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,511,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,63,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,4095,0.11178666353225708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,63,0.3322880069414775
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,2047,0.0682666649421056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,8191,0.20036266247431436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,3,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,16383,0.35140268007914227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,31,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,15,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,7,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,63,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,127,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,32767,0.7115093072255453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,2047,0.11229866743087769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,511,0.04795733094215393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,4095,0.1890986760457357
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,8191,0.34406399726867676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,127,0.4123306671778361
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1023,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,16383,0.7132159868876139
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,255,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,7,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,32767,1.3878614107767742
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1023,0.11878400047620137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,511,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,2047,0.22459733486175537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,4095,0.40516265233357746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,3,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,7,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,8191,0.859818696975708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,63,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,127,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,255,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1023,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,16383,1.7059839566548665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,2047,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,16383,0.04761599997679392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,8191,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,4095,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,32767,0.05273599922657013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,131071,0.09130666653315227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,65535,0.06980266670385997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,7,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1023,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,4095,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,8191,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,2047,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,16383,0.03839999934037527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,32767,0.05649066468079885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,65535,0.08004266520341237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,255,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,131071,0.12066133817036946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,32767,3.3454081217447915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,127,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1023,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,4095,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,16383,0.04454400142033895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,8191,0.03259733319282532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,32767,0.06809600194295247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,65535,0.10717866818110149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,2047,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,7,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,131071,0.18978132804234824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,31,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,63,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,2047,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1023,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,4095,0.1013759970664978
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,511,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,8191,0.19780266284942627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,16383,0.38570666313171387
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,32767,0.7679999669392904
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,65535,1.5131306648254395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,511,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,2047,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1023,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,4095,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,8191,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,16383,0.03908266623814901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,32767,0.05853866537412008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,65535,0.08106666803359985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,131071,0.12731732924779257
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,3,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,7,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,31,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,131071,3.0218238830566406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,15,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,63,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,127,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,511,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,4095,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1023,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,2047,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,8191,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,16383,0.053930665055910744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,32767,0.07441066702206929
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,65535,0.12612266341845194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,131071,0.20172800620396933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,3,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,31,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,15,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,127,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,255,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1023,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,511,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,2047,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,4095,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,8191,0.049322664737701416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,16383,0.06775466601053874
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,32767,0.10786133011182149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,65535,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,7,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,31,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,131071,0.3797333240509033
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,63,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1023,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,511,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,4095,0.10376532872517903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,2047,0.055125330885251365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,8191,0.19933867454528809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,16383,0.38690133889516193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,32767,0.7627092997233073
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,3,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,65535,1.5192747116088867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,7,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,15,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,31,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,63,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,131071,3.018239974975586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,127,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,255,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,511,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1023,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,8191,0.1925119956334432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,16383,0.34457600116729736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,3,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,2047,0.06178133189678192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,15,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,4095,0.10427733262379964
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,63,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,31,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,127,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,7,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,255,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,4095,0.20138667027155557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1023,0.07031466563542683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,8191,0.3500373363494873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,511,0.04949333270390829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,16383,0.7135573228200277
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,2047,0.11639466881752014
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,3,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,15,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,31,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,7,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,63,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,127,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,255,0.040106666584809623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,2047,0.22323199113210043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,4095,0.41864534219106037
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1023,0.12014933427174886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,511,0.07116800049940745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,8191,0.8751786549886068
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,7,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,3,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,15,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,127,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1023,0.1800533334414164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,63,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,2047,0.3696639935175578
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,31,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,16383,1.7220266660054524
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,255,0.055125330885251365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,511,0.09779199957847595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,4095,0.6690133412679037
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,8191,1.4257493019104004
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,8191,0.03310933212439219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,2047,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,4095,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,16383,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,65535,0.11366400122642517
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,32767,0.07389866809050243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,7,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,131071,0.20667733748753866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,255,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,511,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1023,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,4095,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,2047,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,8191,0.04915200173854828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,16383,0.07338666419188182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,32767,0.11281067132949829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,65535,0.19814399878184
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,131071,0.3536213239034017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,63,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,16383,2.8402347564697266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,255,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,511,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,2047,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,4095,0.044031997521718345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1023,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,16383,0.10513066252072652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,32767,0.18978132804234824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,65535,0.3763200044631958
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,7,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,15,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,31,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,131071,0.6517759958902994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,8191,0.06860800087451935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,255,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,63,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1023,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,511,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,2047,0.05717333157857259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,4095,0.10564266641934712
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,8191,0.19933867454528809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,16383,0.38997332255045575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,32767,0.7668053309122721
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,65535,1.522175947825114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,3,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,7,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,15,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,31,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,131071,3.0453761418660483
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,63,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,127,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,255,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,511,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1023,0.06502399841944377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,8191,0.36454399426778156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,3,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,7,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,2047,0.10990933577219646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,15,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,4095,0.19780266284942627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,63,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,127,0.02918400118748347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,31,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1023,0.12475732962290446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,255,0.04505600035190582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,511,0.07253333429495494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,3,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,2047,0.2244266668955485
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,8191,0.8656213283538818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,4095,0.42496001720428467
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,15,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,63,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,31,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,127,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,7,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,2047,0.3556693394978841
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,255,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,3,0.04130133241415024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,511,0.1109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,4095,0.7357439994812012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,8191,1.4438400268554688
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,7,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1023,0.19182932376861572
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,31,0.04181333382924398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,127,0.05529599885145823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,15,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,63,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1023,0.353279987970988
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,255,0.09676800171534221
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,4095,1.3491199811299641
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,3,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,2047,0.6270293394724528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,7,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,511,0.18653867642084757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,15,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,31,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,63,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,127,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,8191,2.671104113260905
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,255,0.040618665516376495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,511,0.06519466638565063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1023,0.11673600474993388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,3,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,7,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,2047,0.2135039965311686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,15,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,31,0.03498666733503342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,127,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,63,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,255,0.06809600194295247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1023,0.19370667139689127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,511,0.11264000336329143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1,0.05222400029500326
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,3,0.05205333232879639
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,7,0.05273599922657013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,15,0.05256533126036326
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,127,0.06877866884072621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,63,0.05973333120346069
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,255,0.11110400160153706
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,31,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,2047,0.7014400164286295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1,0.07338666419188182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,3,0.07372800012429555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,511,0.1890986760457357
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,15,0.07594666878382365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,2047,0.3609600067138672
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,31,0.07884799937407176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,7,0.07543466488520305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,255,0.18995199600855509
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,63,0.08004266520341237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,127,0.09454933802286784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1023,0.368127981821696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,2047,1.3540693918863933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1023,0.6314666668574015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,511,0.35788798332214355
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,3,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,7,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,15,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,63,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,127,0.040448000033696495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,31,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,255,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1,0.053930665055910744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,511,0.10393599669138591
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,3,0.05376000205675761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1023,0.21196800470352173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,7,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,15,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,31,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,63,0.06178133189678192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,127,0.07287466526031494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,255,0.1155413289864858
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1,0.08686932921409607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,511,0.19370667139689127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,3,0.08772266904513042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1023,0.3659093379974365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,7,0.08891733487447102
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,15,0.0897706647713979
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,31,0.09437867005666097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,63,0.0962559978167216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,255,0.19643733898798624
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1,0.1358506679534912
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,127,0.11315199732780457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,7,0.14045866330464682
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,511,0.35020800431569415
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,15,0.14677332838376364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,3,0.13687466581662497
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,31,0.14404267072677612
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,15,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,63,0.1454080045223236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,127,0.19268266359965006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1023,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,2047,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1023,0.7094613711039225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,4095,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,8191,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,65535,0.19114667177200317
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,32767,0.1083733340104421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,131071,0.3510613441467285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1023,1.3470719655354817
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,31,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,127,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,511,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1023,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,2047,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,4095,0.04659200211366018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,8191,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,16383,0.11059199770291646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,32767,0.19182932376861572
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,65535,0.3763200044631958
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,7,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,131071,0.672426700592041
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,15,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,255,0.37444265683492023
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,31,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,63,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,255,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,511,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,2047,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,4095,0.06877866884072621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,8191,0.10666666428248088
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,16383,0.18807466824849448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,32767,0.340992013613383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,511,0.6927359898885092
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,3,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,65535,0.7167999744415283
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,31,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,63,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,16383,0.06587733328342438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,255,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,127,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,131071,1.3887146313985188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,511,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1023,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,2047,0.059903999169667564
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,4095,0.11178666353225708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,8191,0.20957867304484049
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,16383,0.3957759936650594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,32767,0.7995733420054117
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,65535,1.5817386309305828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,131071,3.163818677266439
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,7,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,1,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,3,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,7,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,15,0.04642133414745331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,31,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,63,0.055125330885251365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,255,0.1083733340104421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,127,0.06400000055631001
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,1,0.09062400460243225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,3,0.09113599856694539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,7,0.0936959981918335
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,511,0.18449066082636514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,15,0.09659733374913533
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,31,0.09983999530474345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,63,0.10564266641934712
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,127,0.13158399860064188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,1,0.15530666708946228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,255,0.20514132579167685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,7,0.1646933356920878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,3,0.15752533078193665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,15,0.16827734311421713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,31,0.16827734311421713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,511,0.381440003712972
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,63,0.17595734198888144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,1,0.2609493335088094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,3,0.26641066869099933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,127,0.20565332969029745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,7,0.2805759906768799
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,255,0.36642134189605713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,31,0.2769920031229655
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,15,0.27716267108917236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,511,0.7200427055358887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,63,0.2800640066464742
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,127,0.3461120128631592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,255,0.6534826755523682
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,1,0.07884799937407176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,3,0.07935999830563863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,511,1.3632853825887044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,7,0.08413867155710857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,15,0.08703999718030293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,31,0.09062400460243225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,63,0.09335466225941975
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,127,0.12032000223795573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,1,0.16401066382726034
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,255,0.19490132729212442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,7,0.17783466974894205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,15,0.18039466937383017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,3,0.17032533884048462
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,31,0.18090667327245077
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,63,0.18449066082636514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,127,0.2310826579729716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,1,0.30003199974695843
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,255,0.385535995165507
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,3,0.3130026658376058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,7,0.3341653347015381
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,15,0.31829333305358887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,31,0.31948800881703693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,63,0.3232426643371582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,127,0.3993599812189738
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,1,0.5210453271865845
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,3,0.5360639890034994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,3,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,7,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,255,0.7567359606424967
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,7,0.5439146757125854
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1023,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,15,0.5435733397801717
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,2047,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,4095,0.039594667653242745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,8191,0.06570666531721751
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,16383,0.10598400235176086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,65535,0.34406399726867676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,3,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,7,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,32767,0.19746132691701254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,15,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,31,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,127,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,255,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,63,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,511,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1023,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,2047,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,31,0.5478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,4095,0.06587733328342438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,8191,0.11076266566912334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,32767,0.35020800431569415
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,16383,0.1889280080795288
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,3,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,65535,0.66594131787618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,7,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,63,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,15,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,31,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,127,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,255,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,511,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,2047,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1023,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,511,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,8191,0.18875734011332193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,4095,0.11468799908955891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,63,0.5597866773605347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,16383,0.34781865278879803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,3,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,32767,0.6560426553090414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,15,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,31,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,63,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,127,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,255,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,65535,1.3969066937764485
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1023,0.060415998101234436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,2047,0.1104213297367096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,4095,0.19780266284942627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,511,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,8191,0.40533332029978436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,16383,0.745130697886149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,7,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,32767,1.5831039746602376
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,31,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,7,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,127,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,255,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,511,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1023,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,65535,3.1600640614827475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,2047,0.038058665891488395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,8191,0.11059199770291646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,16383,0.1843199928601583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,32767,0.3397973378499349
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,127,0.6811306476593018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,4095,0.06007466713587443
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,15,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,63,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,255,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,127,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,4095,0.10905599594116211
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,511,0.03054933249950409
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1023,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,8191,0.19131733973821005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,65535,0.6981973648071289
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,16383,0.3476479848225911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,32767,0.6657706499099731
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,3,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,7,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,31,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,15,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,63,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,127,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,255,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,2047,0.1083733340104421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1023,0.05836800237496694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,511,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,4095,0.20087466637293497
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,65535,1.3904213905334473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,8191,0.4092586835225423
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,3,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,7,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,2047,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,15,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,31,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,16383,0.7586133480072021
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,255,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,127,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,32767,1.5907840728759766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1023,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,4095,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,511,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,2047,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,32767,0.05085866649945577
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,65535,0.06434133152167003
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,16383,0.04539733131726583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,131071,0.09284266829490662
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,3,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,8191,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,15,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,127,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,255,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,511,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1023,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,2047,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,8191,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,4095,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,16383,0.03874133278926214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,65535,0.07765333354473114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,32767,0.057855998476346336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,131071,0.12987732887268066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,7,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,15,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,3,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,63,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,31,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,65535,3.1747414271036782
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,255,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,2047,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,4095,0.10103467106819153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,8191,0.19387733936309814
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,32767,0.7613440354665121
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,16383,0.38758401075998944
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,127,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,65535,1.510741392771403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,255,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1023,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,511,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,2047,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,8191,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,4095,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,16383,0.03839999934037527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,32767,0.053077335158983864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,65535,0.08294400076071422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,131071,3.0122667948404946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,131071,0.11929600437482198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,7,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,15,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,3,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,31,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,63,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,127,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,255,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,511,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1023,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,4095,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,8191,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,16383,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,2047,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,32767,0.07423999905586243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,65535,0.1181013286113739
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,3,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,7,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,31,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,131071,0.21401600042978922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,63,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,511,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,2047,0.05495466788609823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,4095,0.1013759970664978
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,8191,0.1971199909845988
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,16383,0.3862186670303345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,32767,0.7618559996287028
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,255,1.4037334124247234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,65535,1.5131306648254395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,7,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,63,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,131071,3.0366719563802085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,511,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1023,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,2047,0.06092800199985504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,16383,0.33894399801890057
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,8191,0.19592533508936563
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,4095,0.10376532872517903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,7,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,15,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,31,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,3,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,127,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,255,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,511,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,32767,0.6932480335235596
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1023,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,4095,0.1930239995320638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,63,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,3,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,16383,0.7096319993336996
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,2047,0.11485866705576579
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,7,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,8191,0.36983466148376465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,32767,1.3796693483988445
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,31,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,255,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,2047,0.2239146629969279
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,4095,0.40857601165771484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1023,0.11195733149846394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,511,0.06468266745408376
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,8191,0.8630613485972086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1023,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,511,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,2047,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,8191,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,4095,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,16383,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,32767,0.06929066777229309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,15,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,65535,0.11281067132949829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,131071,0.19336533546447754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,3,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,7,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,16383,1.709397315979004
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,15,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,31,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,63,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,127,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,255,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,511,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,4095,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,2047,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,16383,0.06911999980608623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,8191,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,32767,0.11332266529401143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,3,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,65535,0.19814399878184
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,131071,0.35549867153167725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,7,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,63,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1023,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,511,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,32767,3.263146718343099
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,2047,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,4095,0.10376532872517903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1023,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,8191,0.20002132654190063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,16383,0.3872426748275757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,127,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,65535,1.508522669474284
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,32767,0.7668053309122721
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,7,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,3,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,15,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,131071,3.044010798136393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,31,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,127,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,63,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,255,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1023,0.06434133152167003
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,511,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,2047,0.11468799908955891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,8191,0.3428693215052287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,16383,0.6934186617533366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,4095,0.18602667252222696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,3,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,7,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,63,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,31,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,127,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,4095,0.4398080110549927
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,15,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,511,0.0718506673971812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,8191,0.8647680282592773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,3,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1023,0.12339199582735698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,255,0.04420266548792521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,7,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,2047,0.22425599892934164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,15,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,31,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,63,0.02611200014750163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,255,0.053930665055910744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,127,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1023,0.19319466749827066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,2047,0.3438933293024699
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,511,0.09727999567985535
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,4095,0.7272106806437174
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,16383,1.7136640548706055
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,8191,1.4387200673421223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,7,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,31,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,15,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,3,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,16383,2.859349250793457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,127,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,63,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,511,0.06451199948787689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,255,0.03925333420435587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1023,0.11400533715883891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,3,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,7,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,2047,0.2141866683959961
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,15,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,31,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,63,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,127,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1023,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,4095,0.4331520001093547
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,255,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,4095,0.7350613276163737
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,3,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,511,0.11212799946467082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,2047,0.3802453279495239
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,31,0.04351999859015147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,63,0.04898133377234141
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,127,0.05580799778302511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,7,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,255,0.09898666540781657
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1023,0.355840007464091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,511,0.176639993985494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,15,0.04369066655635834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,2047,0.6910293102264404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,4095,1.357141335805257
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,3,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,7,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,63,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,127,0.03754666695992152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,31,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,255,0.05870933334032694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,15,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,511,0.10222933689753215
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1,0.05171200136343638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,3,0.05205333232879639
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1023,0.18363734086354574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,7,0.05239466826121012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,15,0.05273599922657013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,2047,0.36983466148376465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,31,0.05222400029500326
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,127,0.06929066777229309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,63,0.059562668204307556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,511,0.19012266397476196
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,255,0.11383466919263203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,3,0.0766293356815974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1,0.07577600081761678
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,15,0.07867733140786488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,7,0.07799466451009114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1023,0.3676160176595052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,63,0.08226133386294048
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,2047,0.6980266571044922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,127,0.09796266754468282
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,31,0.08106666803359985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,255,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,511,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1023,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,2047,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,255,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,8191,0.040448000033696495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,4095,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,16383,0.06382933259010315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,65535,0.19182932376861572
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,3,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,7,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,131071,0.3490133285522461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,15,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,31,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,63,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,127,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,511,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1023,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,2047,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,4095,0.04351999859015147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,8191,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,511,0.36130134264628094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,16383,0.1104213297367096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,32767,0.19285333156585693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,65535,0.35089067618052167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,7,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,3,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,15,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,32767,0.10786133011182149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,31,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,63,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,255,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,511,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,2047,0.05717333157857259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,4095,0.10496000448862712
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1023,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,8191,0.2001919945081075
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,16383,0.3887786865234375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,32767,0.7669760386149088
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,2047,1.245184024175008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,65535,1.5214932759602864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,131071,3.058858553568522
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,131071,0.6698666413625082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1023,0.6360746622085571
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1,0.04454400142033895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,3,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,7,0.04539733131726583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,15,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,31,0.04539733131726583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,63,0.053077335158983864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,127,0.06382933259010315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,255,0.10461866855621338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1,0.08584533135096233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,3,0.0865280032157898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,7,0.0890880028406779
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,511,0.18056533734003702
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,15,0.08994133273760478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,31,0.09386666615804036
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1023,0.3561813433965047
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,63,0.09796266754468282
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,127,0.116565336783727
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1,0.14011733730634054
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,3,0.14114133516947427
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,255,0.1955839991569519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,511,0.34918399651845294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1023,0.6997333367665609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,7,0.1443839967250824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,15,0.15121066570281982
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,31,0.14984533190727234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,127,0.2001919945081075
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,63,0.15428266922632852
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,255,0.3403093417485555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,511,0.6973439852396647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1023,1.3597013155619304
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,1,0.07714133461316426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,3,0.0773119976123174
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,7,0.08055466910203297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,31,0.08874666690826416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,15,0.0820906658967336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,63,0.08925867080688477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,511,0.36181334654490155
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,1,0.15496533115704855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,127,0.10820266604423523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,3,0.15718400478363037
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,7,0.16486400365829468
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,255,0.1890986760457357
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,31,0.17220266660054526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,15,0.1699840029080709
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,63,0.17442133029301962
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,127,0.21486934026082358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,1,0.2701653242111206
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,255,0.38809601465861004
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,7,0.29013333717981976
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,15,0.2879146734873454
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,3,0.2759679953257243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,3,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,7,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,15,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,63,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,511,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1023,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,2047,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,511,0.6736213366190592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,4095,0.03737599899371465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,8191,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,65535,0.34201598167419434
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,32767,0.18722132841746011
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,3,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,7,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,131071,0.7014400164286295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,127,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,63,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,255,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,511,0.02611200014750163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1023,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,2047,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,4095,0.06553600231806438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,8191,0.1088853379090627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,16383,0.1867093245188395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,32767,0.34815998872121173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,16383,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,65535,0.719701369603475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,7,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,3,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,15,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,31,0.2949120004971822
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,31,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,131071,1.3800106048583984
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,63,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,127,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,511,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,255,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,2047,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1023,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,4095,0.10854400197664897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,8191,0.21794132391611734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,63,0.2955946723620097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,16383,0.3985066811243693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,32767,0.8120319843292236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,65535,1.6080212593078613
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,63,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,255,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,511,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,131071,3.191808064778646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,2047,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1023,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,4095,0.06007466713587443
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,8191,0.10376532872517903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,32767,0.3597653309504191
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,16383,0.21964800357818604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,127,0.3638613224029541
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,65535,0.6995626290639242
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,3,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,7,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,15,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,63,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,255,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,127,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,511,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,255,0.7205546696980795
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1023,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,4095,0.11588266491889954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,131071,1.3714772860209148
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,2047,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,16383,0.4089173475901286
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,32767,0.8209066390991211
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,15,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,7,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,65535,1.6310613950093586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,63,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,127,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,511,0.01757866640885671
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,4095,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,2047,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1023,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,8191,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,255,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,32767,0.050517335534095764
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,16383,0.04642133414745331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,65535,0.06297599772612254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,131071,0.09284266829490662
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,3,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,7,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,15,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,31,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,255,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,63,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,2047,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,4095,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,8191,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,131071,3.2481279373168945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,16383,0.37956265608469647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,32767,0.7598079840342203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,15,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,65535,1.509376049041748
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,31,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,63,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,127,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,511,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,131071,3.0242134730021157
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1023,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,2047,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,4095,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,8191,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,16383,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,32767,0.05341866612434387
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,65535,0.0766293356815974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,131071,0.12851199507713318
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,7,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,8191,0.20957867304484049
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,15,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,31,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,63,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,511,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,255,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,2047,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,4095,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,8191,0.19404800732930502
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,16383,0.38417065143585205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,32767,0.7664639949798584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,3,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,65535,1.5182506243387859
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,7,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,31,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,131071,3.0247252782185874
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,63,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,127,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,255,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,511,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1023,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,8191,0.183296004931132
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,2047,0.060415998101234436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,4095,0.10342400272687276
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,511,1.3844480514526367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,16383,0.36300798257191974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,3,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,31,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,32767,0.6987093289693197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,7,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,15,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,127,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,255,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,65535,1.3649919827779133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,4095,0.20462934176127115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1023,0.05870933334032694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,511,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,8191,0.39048532644907635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,16383,0.8110079765319824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,15,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,2047,0.10752000411351521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,63,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1023,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,2047,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,4095,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,8191,0.03259733319282532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,7,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,16383,0.04795733094215393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,65535,0.1114453375339508
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,32767,0.06860800087451935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,131071,0.19438934326171875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,7,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,15,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,31,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,63,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,511,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,4095,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,8191,0.19746132691701254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,2047,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,32767,1.6059734026590984
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,16383,0.38758401075998944
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,32767,0.7611733277638754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,65535,1.5153493881225586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,65535,3.20034122467041
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,131071,3.03872013092041
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,15,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,7,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,31,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,3,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,63,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,255,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,511,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1023,0.0628053347269694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,8191,0.36351998647054035
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,127,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,2047,0.10734933614730835
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,3,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,16383,0.6959786415100098
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,15,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,32767,1.3631146748860676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,4095,0.18312533696492514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,31,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,63,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,255,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,127,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,511,0.06485333542029063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1023,0.11434666315714519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,4095,0.43383467197418213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,8191,0.8541866938273112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,2047,0.2141866683959961
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,15,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,3,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,31,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,7,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,16383,1.7064959208170574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,127,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,255,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,511,0.06434133152167003
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,63,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1023,0.11571199695269267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,32767,3.4034347534179688
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,3,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,8191,0.8487253189086914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,7,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,15,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,2047,0.22459733486175537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,63,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,4095,0.434005339940389
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,127,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,255,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1023,0.18039466937383017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,31,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,511,0.09864532947540283
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,4095,0.6784000396728516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,2047,0.3466240167617798
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,8191,1.4388906160990398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,7,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,3,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,15,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,31,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,63,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,255,0.05819733440876007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1023,0.18244266510009766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,511,0.10786133011182149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,127,0.03669333209594091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,3,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,7,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,2047,0.35089067618052167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,31,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,127,0.0576853354771932
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,15,0.04351999859015147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,63,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,7,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,255,0.10035199920336406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,15,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,31,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,127,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,255,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,4095,0.7205546696980795
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,511,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1023,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,63,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,511,0.18414932489395142
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,4095,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,8191,0.04027733455101649
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,16383,0.06365866462389629
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1023,0.3319466710090637
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,65535,0.18824533621470133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,131071,0.37085866928100586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,15,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,31,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,63,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1023,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,2047,0.05495466788609823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,4095,0.10410666465759277
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,8191,0.19882667064666748
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,16383,0.38946131865183514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,32767,0.7664639949798584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,4095,1.3574825922648113
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,32767,0.10734933614730835
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,65535,1.5162026087443035
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,131071,3.056469281514486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,7,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,2047,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,2047,0.6900053024291992
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1,0.04437333345413208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,3,0.04505600035190582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,7,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,15,0.04539733131726583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,31,0.045567999283472695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,63,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,127,0.06144000093142191
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1,0.07492266595363617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,255,0.10427733262379964
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,511,0.18961066007614136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,3,0.07577600081761678
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,7,0.07799466451009114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1023,0.33348266283671063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,15,0.07765333354473114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,2047,0.6840319633483887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,31,0.08072533210118611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,63,0.08482133348782857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,127,0.10291199882825215
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,255,0.183296004931132
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,511,0.33689598242441815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1023,0.6468266646067301
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,2047,1.3504853248596191
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1,0.0769706666469574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,3,0.07714133461316426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,7,0.07987200220425923
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,15,0.08038400113582611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,31,0.08738133311271667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,63,0.08891733487447102
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,127,0.10444800059000652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1,0.1397760013739268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,511,0.33740798632303876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,255,0.187391996383667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,3,0.14079999923706055
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1023,0.6867626508076986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,7,0.144896000623703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,3,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,15,0.15018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,7,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,15,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,63,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,31,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,127,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,31,0.15428266922632852
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,255,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,511,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1023,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,4095,0.03857066730658213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,8191,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,2047,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,16383,0.10513066252072652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,32767,0.18500266472498575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,65535,0.36471466223398846
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,3,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,7,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,15,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,31,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,63,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,131071,0.6993920008341471
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,127,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,511,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1023,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,2047,0.05717333157857259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,4095,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,8191,0.20206934213638306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,16383,0.39150933424631756
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,63,0.1565013329188029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,32767,0.7749973138173422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,65535,1.542143980662028
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,7,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,3,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,31,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,63,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,127,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,127,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,131071,3.0697813034057617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,511,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1023,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,2047,0.0576853354771932
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,8191,0.202239990234375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,4095,0.10700800021489461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,32767,0.7765333652496338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,255,0.3474773168563843
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,3,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,16383,0.3933866818745931
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,7,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,31,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,63,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,65535,1.5499946276346843
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,127,0.010581333190202713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,511,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,255,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,2047,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,4095,0.10069333513577779
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,8191,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,131071,3.079338709513346
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,16383,0.37956265608469647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,32767,0.7538346449534098
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,3,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,7,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,15,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,65535,1.5090346336364746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,31,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,63,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,255,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,511,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,131071,3.0219945907592773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,15,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,2047,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,8191,0.1945599913597107
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,4095,0.10086400310198466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,16383,0.3797333240509033
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,32767,0.7639040152231852
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,511,0.6985387007395426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,65535,1.5121067365010579
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,3,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,7,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,15,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,131071,3.021482785542806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,31,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,63,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,127,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,511,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1023,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,8191,0.21026132504145303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,2047,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1023,1.3513387044270833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,16383,0.40755200386047363
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,4095,0.11127466956774394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,15,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,31,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,32767,0.8156159718831381
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,63,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,255,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,3,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,511,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1023,0.0314026673634847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,2047,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,4095,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,8191,0.19387733936309814
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,16383,0.3863893349965413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,32767,0.7659520308176676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,65535,1.630890687306722
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,65535,1.521663983662923
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,131071,3.2467625935872397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,131071,3.0276266733805337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,7,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,31,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,63,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,127,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,3,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,255,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,255,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,15,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,511,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1023,0.05819733440876007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,8191,0.38946131865183514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,2047,0.10598400235176086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,16383,0.8046933015187582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,32767,1.5873707135518391
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,3,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,4095,0.20138667027155557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,15,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,7,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,31,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,63,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,127,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,511,0.062122667829195656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,65535,3.151871999104818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1023,0.11195733149846394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,8191,0.7884799639383951
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,255,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,2047,0.2121386726697286
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,7,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,4095,0.4094293514887492
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,7,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,3,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,16383,1.6948906580607097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,15,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,63,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,255,0.05614933371543884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,511,0.10086400310198466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,7,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,15,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,3,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,31,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,63,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,127,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,255,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,511,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,127,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1023,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,2047,0.05614933371543884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1023,0.18158932526906332
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,8191,1.4158506393432617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,4095,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,8191,0.19985065857569376
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,16383,0.38809601465861004
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,65535,1.521663983662923
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,32767,0.7674880027770996
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,2047,0.3684693177541097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,131071,3.0371840794881186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,4095,0.7176533540089926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,31,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,3,0.04266666869322459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,7,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,15,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,63,0.05017599960168203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,31,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,255,0.09966933727264404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,127,0.05905066430568695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,511,0.18619734048843384
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1023,0.33075199524561566
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1,0.07423999905586243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,2047,0.6411946614583334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,4095,1.3383679389953613
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,3,0.07560533285140991
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,7,0.07765333354473114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,15,0.07765333354473114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,31,0.0846506655216217
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,7,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,15,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,63,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,31,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,127,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,255,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,511,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1023,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,2047,0.055125330885251365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,4095,0.10359467069307964
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,8191,0.20036266247431436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,16383,0.3906559944152832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,32767,0.7714133262634277
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,65535,1.5325867335001628
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,63,0.08550399541854858
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,131071,3.0580053329467773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,127,0.10188800096511841
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,255,0.18175999323527017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,511,0.33399466673533124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1023,0.6423893372217814
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,2047,1.3395627339680989
