framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,1,128,1,float16,float16,0,96.47487386067708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,4,128,1,float16,float16,0,97.50966389973958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,64,128,1,float16,float16,0,99.17015584309895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,2,128,1,float16,float16,0,97.69143676757812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,64,128,1,float16,float16,0,49.57408650716146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,1,128,1,float16,float16,0,48.239959716796875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,2,128,1,float16,float16,0,48.87545267740885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,8,128,1,float16,float16,0,97.17519124348958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,64,128,1,float16,float16,0,25.21924336751302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,64,1,128,1,float16,float16,0,196.39986165364584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,4,128,1,float16,float16,0,48.62355041503906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,64,2,128,1,float16,float16,0,195.060546875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,8,128,1,float16,float16,0,48.704630533854164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,1,128,1,float16,float16,0,24.16749318440755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,2,128,1,float16,float16,0,24.562037150065105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,4,128,1,float16,float16,0,24.661471048990887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,8,128,1,float16,float16,0,24.33367411295573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,64,8,128,1,float16,float16,0,196.41984049479166
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,64,4,128,1,float16,float16,0,196.71270751953125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,1,128,1,float16,float16,0,54.60668436686198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,64,128,1,float16,float16,0,57.93043009440104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,2,128,1,float16,float16,0,55.51258341471354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,4,128,1,float16,float16,0,55.84669494628906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,64,128,1,float16,float16,0,28.9026616414388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,1,128,1,float16,float16,0,27.82349395751953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,64,1,128,1,float16,float16,0,110.58707682291667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,2,128,1,float16,float16,0,27.797627766927082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,64,2,128,1,float16,float16,0,110.29703776041667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,64,4,128,1,float16,float16,0,110.50828043619792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,1,128,1,float16,float16,0,13.879578908284506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,64,128,1,float16,float16,0,14.370138804117838
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,64,8,128,1,float16,float16,0,111.82188924153645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,8,128,1,float16,float16,0,55.2839101155599
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,4,128,1,float16,float16,0,27.921483357747395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,8,128,1,float16,float16,0,27.946571350097656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,2,128,1,float16,float16,0,13.914661407470703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,4,128,1,float16,float16,0,13.816543579101562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,8,128,1,float16,float16,0,14.07253901163737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,1,128,1,float16,float16,0,38.18160502115885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,64,128,1,float16,float16,0,40.41069793701172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,2,128,1,float16,float16,0,38.65447489420573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,4,128,1,float16,float16,0,39.10241190592448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,1,128,1,float16,float16,0,19.11799494425456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,64,128,1,float16,float16,0,20.358331044514973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,2,128,1,float16,float16,0,19.486955006917317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,64,1,128,1,float16,float16,0,77.63909912109375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,64,4,128,1,float16,float16,0,77.53331502278645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,64,2,128,1,float16,float16,0,77.77776590983073
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,64,128,1,float16,float16,0,9.973482767740885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,1,128,1,float16,float16,0,9.567461649576822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,8,128,1,float16,float16,0,38.8387196858724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,64,8,128,1,float16,float16,0,79.14306640625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,4,128,1,float16,float16,0,19.27779261271159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,2,128,1,float16,float16,0,9.544719696044922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,4,128,1,float16,float16,0,9.628421147664389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,8,128,1,float16,float16,0,19.525845845540363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,8,128,1,float16,float16,0,9.777178446451822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,1,128,1,float16,float16,0,49.8181406656901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,2,128,1,float16,float16,0,50.27978515625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,64,128,1,float16,float16,0,52.03602600097656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,4,128,1,float16,float16,0,50.57872517903646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,64,128,1,float16,float16,0,26.340382893880207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,1,128,1,float16,float16,0,24.72290547688802
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,2,128,1,float16,float16,0,25.159268697102863
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,64,128,1,float16,float16,0,13.175771077473959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,64,2,128,1,float16,float16,0,99.91770426432292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,64,1,128,1,float16,float16,0,100.3043721516927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,64,4,128,1,float16,float16,0,102.38901774088542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,64,8,128,1,float16,float16,0,100.40799967447917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,8,128,1,float16,float16,0,50.674540201822914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,8,128,1,float16,float16,0,25.057846069335938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,4,128,1,float16,float16,0,25.29119364420573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,1,128,1,float16,float16,0,12.487786610921225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,64,128,1,float16,float16,0,6.333445231119792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,1,128,1,float16,float16,0,6.158762613932292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,2,128,1,float16,float16,0,12.330331166585287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,4,128,1,float16,float16,0,12.574427286783854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,8,128,1,float16,float16,0,12.694042205810547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,4,128,1,float16,float16,0,6.190954844156901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,2,128,1,float16,float16,0,6.091941197713216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,8,128,1,float16,float16,0,6.203562418619792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,2,128,1,float16,float16,0,28.345535278320312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,1,128,1,float16,float16,0,28.693013509114582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,4,128,1,float16,float16,0,29.011423746744793
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,64,128,1,float16,float16,0,30.7369867960612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,64,128,1,float16,float16,0,15.209584554036459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,1,128,1,float16,float16,0,14.258783976236979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,2,128,1,float16,float16,0,14.386826833089193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,64,1,128,1,float16,float16,0,56.574371337890625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,64,2,128,1,float16,float16,0,57.73511250813802
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,64,128,1,float16,float16,0,7.607237497965495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,64,4,128,1,float16,float16,0,57.43000284830729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,64,8,128,1,float16,float16,0,57.94215393066406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,8,128,1,float16,float16,0,28.725382486979168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,1,128,1,float16,float16,0,6.841621398925781
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,2,128,1,float16,float16,0,7.0379683176676435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,4,128,1,float16,float16,0,14.416917165120443
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,64,128,1,float16,float16,0,3.6663198471069336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,8,128,1,float16,float16,0,14.536842346191406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,1,128,1,float16,float16,0,3.433984120686849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,4,128,1,float16,float16,0,7.216213226318359
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,2,128,1,float16,float16,0,3.41155211130778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,8,128,1,float16,float16,0,7.00265630086263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,4,128,1,float16,float16,0,3.462106704711914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,8,128,1,float16,float16,0,3.541935920715332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,1,128,1,float16,float16,0,26.005828857421875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,2,128,1,float16,float16,0,26.474217732747395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,64,128,1,float16,float16,0,28.384933471679688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,64,128,1,float16,float16,0,14.146709442138672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,1,128,1,float16,float16,0,12.883312225341797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,4,128,1,float16,float16,0,26.38201649983724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,64,2,128,1,float16,float16,0,52.59034729003906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,64,1,128,1,float16,float16,0,51.962493896484375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,64,4,128,1,float16,float16,0,54.03999328613281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,8,128,1,float16,float16,0,26.756805419921875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,64,8,128,1,float16,float16,0,53.13113911946615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,64,128,1,float16,float16,0,7.041872024536133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,2,128,1,float16,float16,0,13.166239420572916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,1,128,1,float16,float16,0,6.307461420694987
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,4,128,1,float16,float16,0,13.17074712117513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,2,128,1,float16,float16,0,6.504458745320638
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,1,128,1,float16,float16,0,3.1230080922444663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,64,128,1,float16,float16,0,3.463103930155436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,8,128,1,float16,float16,0,12.914629618326822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,4,128,1,float16,float16,0,6.46780268351237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,2,128,1,float16,float16,0,3.141813278198242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,4,128,1,float16,float16,0,3.1518932978312173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,8,128,1,float16,float16,0,3.2664852142333984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,8,128,1,float16,float16,0,6.306442896525065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,64,128,1,float16,float16,0,1.731946627298991
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,1,128,1,float16,float16,0,1.5931092898050945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,4,128,1,float16,float16,0,1.7242666880289714
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,8,128,1,float16,float16,0,1.6383253733317058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,2,128,1,float16,float16,0,1.6086079279581706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,2,128,1,float16,float16,0,15.206106821695963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,4,128,1,float16,float16,0,15.396293640136719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,1,128,1,float16,float16,0,15.22876230875651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,64,128,1,float16,float16,0,16.953418731689453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,1,128,1,float16,float16,0,7.402069091796875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,64,128,1,float16,float16,0,8.296469370524088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,64,2,128,1,float16,float16,0,30.98973846435547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,2,128,1,float16,float16,0,7.416757583618164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,64,4,128,1,float16,float16,0,30.830970764160156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,64,1,128,1,float16,float16,0,30.076820373535156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,64,8,128,1,float16,float16,0,31.456095377604168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,8,128,1,float16,float16,0,15.586538950602213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,4,128,1,float16,float16,0,7.681498845418294
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,64,128,1,float16,float16,0,4.1552480061848955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,8,128,1,float16,float16,0,7.552890777587891
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,64,128,1,float16,float16,0,2.04477326075236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,2,128,1,float16,float16,0,3.5819838841756186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,1,128,1,float16,float16,0,1.833994706471761
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,4,128,1,float16,float16,0,3.6889012654622397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,8,128,1,float16,float16,0,3.7218348185221353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,1,128,1,float16,float16,0,3.563770612080892
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,4,128,1,float16,float16,0,1.8766345977783203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,2,128,1,float16,float16,0,1.847343921661377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,2,128,1,float16,float16,0,0.9686933358510336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,1,128,1,float16,float16,0,0.934874693552653
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,4,128,1,float16,float16,0,0.9790186882019043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,64,128,1,float16,float16,0,1.0694506963094075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,8,128,1,float16,float16,0,0.9800159931182861
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,8,128,1,float16,float16,0,1.8447306950887044
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,1,128,1,float16,float16,0,14.11233647664388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,4,128,1,float16,float16,0,14.612901051839193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,2,128,1,float16,float16,0,14.486495971679688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,64,128,1,float16,float16,0,16.557520548502605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,64,128,1,float16,float16,0,8.211594899495443
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,1,128,1,float16,float16,0,6.860432306925456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,2,128,1,float16,float16,0,6.985455830891927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,64,1,128,1,float16,float16,0,28.16455586751302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,64,4,128,1,float16,float16,0,29.419151306152344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,64,8,128,1,float16,float16,0,29.894564310709637
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,8,128,1,float16,float16,0,14.633733113606771
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,64,128,1,float16,float16,0,3.9649387995402017
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,2,128,1,float16,float16,0,3.4689226150512695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,1,128,1,float16,float16,0,3.3065973917643228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,64,2,128,1,float16,float16,0,29.16979726155599
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,4,128,1,float16,float16,0,7.098165512084961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,1,128,1,float16,float16,0,1.7214080492655437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,8,128,1,float16,float16,0,7.251295725504558
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,64,128,1,float16,float16,0,2.011280059814453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,2,128,1,float16,float16,0,1.677552064259847
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,8,128,1,float16,float16,0,3.5456212361653647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,8,128,1,float16,float16,0,1.8038825988769531
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,64,128,1,float16,float16,0,0.5428266525268555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,2,128,1,float16,float16,0,0.8685759703318278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,64,128,1,float16,float16,0,1.0080640316009521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,4,128,1,float16,float16,0,0.8895359834035238
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,8,128,1,float16,float16,0,0.9093706607818604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,1,128,1,float16,float16,0,0.8801493644714355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,4,128,1,float16,float16,0,3.407408078511556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,2,128,1,float16,float16,0,0.45610666275024414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,4,128,1,float16,float16,0,1.7329386075337727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,8,128,1,float16,float16,0,0.46984533468882245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,4,128,1,float16,float16,0,0.46883734067281085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,1,128,1,float16,float16,0,0.44555731614430744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,64,128,1,float16,float16,0,9.95189348856608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,1,128,1,float16,float16,0,8.016213099161783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,64,8,128,1,float16,float16,0,17.678911844889324
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,2,128,1,float16,float16,0,8.261221567789713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,4,128,1,float16,float16,0,8.125296274820963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,64,4,128,1,float16,float16,0,17.60485331217448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,64,128,1,float16,float16,0,5.016325314839681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,64,1,128,1,float16,float16,0,16.668229420979817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,8,128,1,float16,float16,0,8.75165875752767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,1,128,1,float16,float16,0,3.9057439168294272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,2,128,1,float16,float16,0,3.9584213892618814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,1,128,1,float16,float16,0,1.968725363413493
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,8,128,1,float16,float16,0,4.162634531656901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,4,128,1,float16,float16,0,4.14464537302653
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,2,128,1,float16,float16,0,2.078202724456787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,64,128,1,float16,float16,0,2.4736107190450034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,4,128,1,float16,float16,0,2.068138599395752
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,64,2,128,1,float16,float16,0,16.940938313802082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,2,128,1,float16,float16,0,1.001146634419759
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,4,128,1,float16,float16,0,1.0412480036417644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,64,128,1,float16,float16,0,1.2221333185831706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,1,128,1,float16,float16,0,0.9846719900767008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,8,128,1,float16,float16,0,1.0476426283518474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,8,128,1,float16,float16,0,2.055898666381836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,1,128,1,float16,float16,0,0.5105173190434774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,2,128,1,float16,float16,0,0.2783733407656352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,4,128,1,float16,float16,0,0.5438453356424967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,2,128,1,float16,float16,0,0.522874673207601
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,1,128,1,float16,float16,0,0.27459200223286945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,64,128,1,float16,float16,0,0.3303999900817871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,8,128,1,float16,float16,0,0.5446026722590128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,4,128,1,float16,float16,0,0.30505599578221637
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,8,128,1,float16,float16,0,0.2886773347854614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,64,128,1,float16,float16,0,0.6302773157755533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,1,128,1,float16,float16,0,7.598208109537761
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,2,128,1,float16,float16,0,8.272783915201822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,64,128,1,float16,float16,0,10.42690658569336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,4,128,1,float16,float16,0,8.700063705444336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,64,8,128,1,float16,float16,0,17.60647964477539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,64,1,128,1,float16,float16,0,16.34447479248047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,1,128,1,float16,float16,0,3.7958720525105796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,8,128,1,float16,float16,0,8.556208292643229
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,64,128,1,float16,float16,0,5.030218760172526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,2,128,1,float16,float16,0,3.9263998667399087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,64,2,128,1,float16,float16,0,16.561541239420574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,1,128,1,float16,float16,0,2.006122589111328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,64,128,1,float16,float16,0,2.5353066126505532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,64,128,1,float16,float16,0,1.2598133087158203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,8,128,1,float16,float16,0,4.153002738952637
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,2,128,1,float16,float16,0,1.9388160705566406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,4,128,1,float16,float16,0,2.162933349609375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,4,128,1,float16,float16,0,4.146021207173665
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,8,128,1,float16,float16,0,2.082314650217692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,1,128,1,float16,float16,0,0.9508480230967203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,2,128,1,float16,float16,0,0.9789439837137858
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,2,128,1,float16,float16,0,0.5049013296763102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,64,128,1,float16,float16,0,0.6401813427607218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,8,128,1,float16,float16,0,1.034442663192749
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,4,128,1,float16,float16,0,0.5298826694488525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,1,128,1,float16,float16,0,0.49259201685587567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,4,128,1,float16,float16,0,1.0341599782307942
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,64,128,1,float16,float16,0,0.3370399872461955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,8,128,1,float16,float16,0,0.5343146721522013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,1,128,1,float16,float16,0,0.26266666253407794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,2,128,1,float16,float16,0,0.14664000272750854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,2,128,1,float16,float16,0,0.26527466376622516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,1,128,1,float16,float16,0,0.14436800281206766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,4,128,1,float16,float16,0,0.27986133098602295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,4,128,1,float16,float16,0,0.15446399648984274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,64,128,1,float16,float16,0,0.1837973395983378
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,8,128,1,float16,float16,0,0.2811573346455892
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,8,128,1,float16,float16,0,0.15430933237075806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,64,4,128,1,float16,float16,0,17.611722310384113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,1,128,1,float16,float16,0,5.035077412923177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,2,128,1,float16,float16,0,5.187946637471517
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,64,128,1,float16,float16,0,7.4738508860270185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,64,1,128,1,float16,float16,0,10.505119959513346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,1,128,1,float16,float16,0,2.4211146036783853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,64,128,1,float16,float16,0,3.6564480463663735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,64,2,128,1,float16,float16,0,10.73196792602539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,4,128,1,float16,float16,0,5.544805526733398
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,2,128,1,float16,float16,0,2.5469226837158203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,4,128,1,float16,float16,0,2.684848149617513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,2,128,1,float16,float16,0,1.2518719832102458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,64,128,1,float16,float16,0,1.8373066584269206
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,1,128,1,float16,float16,0,1.1946773529052734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,4,128,1,float16,float16,0,1.3420319557189941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,8,128,1,float16,float16,0,5.562976201375325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,64,128,1,float16,float16,0,0.9222933451334635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,1,128,1,float16,float16,0,0.6082293192545573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,2,128,1,float16,float16,0,0.6307520071665446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,8,128,1,float16,float16,0,2.71726926167806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,64,128,1,float16,float16,0,0.4713653326034546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,8,128,1,float16,float16,0,1.3528800010681152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,1,128,1,float16,float16,0,0.3125813404719035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,64,4,128,1,float16,float16,0,11.551663716634115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,4,128,1,float16,float16,0,0.35043732325236004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,8,128,1,float16,float16,0,0.6963466803232828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,64,128,1,float16,float16,0,0.24582932392756143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,8,128,1,float16,float16,0,0.3545120159784953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,2,128,1,float16,float16,0,0.17322667439778647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,8,128,1,float16,float16,0,0.1876373291015625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,1,128,1,float16,float16,0,0.0944053332010905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,4,128,1,float16,float16,0,0.18522133429845175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,2,128,1,float16,float16,0,0.3306666612625122
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,64,128,1,float16,float16,0,0.13249599933624268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,2,128,1,float16,float16,0,0.09737599889437358
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,4,128,1,float16,float16,0,0.10156800349553426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,8,128,1,float16,float16,0,0.10442133744557698
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,64,128,1,float16,float16,0,0.07755200068155925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,4,128,1,float16,float16,0,0.6898240248362223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,64,8,128,1,float16,float16,0,11.715520222981771
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,1,128,1,float16,float16,0,0.05454400181770325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,2,128,1,float16,float16,0,0.05630933245023092
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,4,128,1,float16,float16,0,0.05906666815280914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,8,128,1,float16,float16,0,0.058389330903689064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,1,128,1,float16,float16,0,0.1683786710103353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,1,128,1,float16,float16,0,1.7002239227294922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,64,128,1,float16,float16,0,2.9629653294881186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,2,128,1,float16,float16,0,1.8006985982259114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,64,1,128,1,float16,float16,0,3.4636265436808267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,64,2,128,1,float16,float16,0,3.6774346033732095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,1,128,1,float16,float16,0,0.8284800052642822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,4,128,1,float16,float16,0,2.000234603881836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,64,128,1,float16,float16,0,1.4881226221720378
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,64,4,128,1,float16,float16,0,4.0658613840738935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,2,128,1,float16,float16,0,0.8907093207041422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,64,128,1,float16,float16,0,0.7579147020975748
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,8,128,1,float16,float16,0,1.0084373156229656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,1,128,1,float16,float16,0,0.4296693404515584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,4,128,1,float16,float16,0,0.9922560056050619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,64,128,1,float16,float16,0,0.38344534238179523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,2,128,1,float16,float16,0,0.45340800285339355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,4,128,1,float16,float16,0,0.49715201059977215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,2,128,1,float16,float16,0,0.2319413423538208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,8,128,1,float16,float16,0,2.0292107264200845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,8,128,1,float16,float16,0,0.5084799925486246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,1,128,1,float16,float16,0,0.21812800566355386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,4,128,1,float16,float16,0,0.2595253388086955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,8,128,1,float16,float16,0,0.2628426750500997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,1,128,1,float16,float16,0,0.11785067121187846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,8,128,1,float16,float16,0,0.13758933544158936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,64,128,1,float16,float16,0,0.20561067263285318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,64,8,128,1,float16,float16,0,4.07637882232666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,4,128,1,float16,float16,0,0.13660800457000732
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,2,128,1,float16,float16,0,0.06955199937025706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,64,128,1,float16,float16,0,0.10732799768447876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,1,128,1,float16,float16,0,0.06785599887371063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,4,128,1,float16,float16,0,0.07518933216730754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,2,128,1,float16,float16,0,0.04022400081157684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,64,128,1,float16,float16,0,0.0628959983587265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,8,128,1,float16,float16,0,0.042912001411120095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,8,128,1,float16,float16,0,0.07663999994595845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,4,128,1,float16,float16,0,0.042170668641726174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,1,128,1,float16,float16,0,0.026101333399613697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,2,128,1,float16,float16,0,0.12282666563987732
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,64,128,1,float16,float16,0,0.03257599969704946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,2,128,1,float16,float16,0,0.02624533325433731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,4,128,1,float16,float16,0,0.02789866675933202
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,8,128,1,float16,float16,0,0.0276053324341774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,1,128,1,float16,float16,0,0.6378346681594849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,2,128,1,float16,float16,0,0.6951573689778646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,64,128,1,float16,float16,0,1.3968799908955891
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,64,1,128,1,float16,float16,0,1.5030506451924641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,1,128,1,float16,float16,0,0.03904533386230469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,64,2,128,1,float16,float16,0,1.5631146430969238
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,2,128,1,float16,float16,0,0.34833065668741864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,1,128,1,float16,float16,0,0.321941335995992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,64,128,1,float16,float16,0,0.7051200071970621
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,4,128,1,float16,float16,0,0.4033386707305908
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,4,128,1,float16,float16,0,0.8047466278076172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,8,128,1,float16,float16,0,0.8207146326700846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,1,128,1,float16,float16,0,0.1664906640847524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,2,128,1,float16,float16,0,0.18075199921925864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,8,128,1,float16,float16,0,0.4101333220799764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,64,128,1,float16,float16,0,0.36128000418345135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,4,128,1,float16,float16,0,0.2060906688372294
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,64,4,128,1,float16,float16,0,1.8057066599527996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,8,128,1,float16,float16,0,0.21120532353719076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,1,128,1,float16,float16,0,0.09187199672063191
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,8,128,1,float16,float16,0,0.11397332946459453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,2,128,1,float16,float16,0,0.09798399607340495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,64,128,1,float16,float16,0,0.19197867314020792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,1,128,1,float16,float16,0,0.05538133283456167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,4,128,1,float16,float16,0,0.10984533031781514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,64,128,1,float16,float16,0,0.10389332969983418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,2,128,1,float16,float16,0,0.05706666906674703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,64,128,1,float16,float16,0,0.05818133552869161
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,4,128,1,float16,float16,0,0.06363733112812042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,8,128,1,float16,float16,0,0.06509866813818614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,64,8,128,1,float16,float16,0,1.8290400505065918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,2,128,1,float16,float16,0,0.03138133386770884
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,1,128,1,float16,float16,0,0.030618667602539062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,64,128,1,float16,float16,0,0.028058665494124096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,4,128,1,float16,float16,0,0.033258666594823204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,1,128,1,float16,float16,0,0.02180800090233485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,8,128,1,float16,float16,0,0.03414933383464813
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,2,128,1,float16,float16,0,0.021375998854637146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,8,128,1,float16,float16,0,0.02310933421055476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,64,128,1,float16,float16,0,0.01838933303952217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,2,128,1,float16,float16,0,0.01580799991885821
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,1,128,1,float16,float16,0,0.01639466608564059
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,4,128,1,float16,float16,0,0.016645333419243496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,8,128,1,float16,float16,0,0.016469333320856094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,4,128,1,float16,float16,0,0.022677332162857056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,1,128,1,float16,float16,0,0.31436800956726074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,2,128,1,float16,float16,0,0.338922659556071
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,4,128,1,float16,float16,0,0.3949813445409139
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,64,2,128,1,float16,float16,0,0.6706133683522543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,64,128,1,float16,float16,0,0.3489866654078166
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,8,128,1,float16,float16,0,0.4031253258387248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,64,4,128,1,float16,float16,0,0.7774133682250977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,8,128,1,float16,float16,0,0.20781334241231283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,64,1,128,1,float16,float16,0,0.6175626516342163
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,2,128,1,float16,float16,0,0.1805386741956075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,4,128,1,float16,float16,0,0.20415999492009482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,1,128,1,float16,float16,0,0.16682666540145874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,64,8,128,1,float16,float16,0,0.7960853576660156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,1,128,1,float16,float16,0,0.09195199608802795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,64,128,1,float16,float16,0,0.18630399306615195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,64,128,1,float16,float16,0,0.10055466492970784
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,4,128,1,float16,float16,0,0.10986666878064473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,8,128,1,float16,float16,0,0.11174933115641277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,2,128,1,float16,float16,0,0.05597866574923197
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,64,128,1,float16,float16,0,0.6984639962514242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,64,128,1,float16,float16,0,0.057034666339556374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,8,128,1,float16,float16,0,0.06260266900062561
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,2,128,1,float16,float16,0,0.030437332888444264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,1,128,1,float16,float16,0,0.02865600089232127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,1,128,1,float16,float16,0,0.02056533346573512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,64,128,1,float16,float16,0,0.028277332584063213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,8,128,1,float16,float16,0,0.03262399882078171
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,4,128,1,float16,float16,0,0.022319999833901722
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,2,128,1,float16,float16,0,0.020848001043001812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,8,128,1,float16,float16,0,0.022384000321229298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,64,128,1,float16,float16,0,0.017935999979575474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,4,128,1,float16,float16,0,0.06227200229962667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,1,128,1,float16,float16,0,0.015418666104475657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,2,128,1,float16,float16,0,0.015626666446526844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,4,128,1,float16,float16,0,0.015909332782030106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,1,128,1,float16,float16,0,0.012074666718641916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,8,128,1,float16,float16,0,0.01632533346613248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,2,128,1,float16,float16,0,0.012309333930412928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,8,128,1,float16,float16,0,0.012432000289360682
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,4,128,1,float16,float16,0,0.012256000190973282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,64,128,1,float16,float16,0,0.013573333621025085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,2,128,1,float16,float16,0,0.09888000289599101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,1,128,1,float16,float16,0,0.29721599817276
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,1,128,1,float16,float16,0,0.05300266544024149
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,4,128,1,float16,float16,0,0.03187733391920725
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,2,128,1,float16,float16,0,0.30867733558019
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,4,128,1,float16,float16,0,0.3312319914499919
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,64,4,128,1,float16,float16,0,0.6510399977366129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,64,128,1,float16,float16,0,0.22675732771555582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,64,8,128,1,float16,float16,0,0.6549973487854004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,64,2,128,1,float16,float16,0,0.6051733493804932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,1,128,1,float16,float16,0,0.1534826656182607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,64,1,128,1,float16,float16,0,0.5827946662902832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,64,128,1,float16,float16,0,0.44686933358510333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,1,128,1,float16,float16,0,0.08260266482830048
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,8,128,1,float16,float16,0,0.33374933401743573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,2,128,1,float16,float16,0,0.1591039995352427
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,8,128,1,float16,float16,0,0.09265066186587016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,4,128,1,float16,float16,0,0.17330666383107504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,2,128,1,float16,float16,0,0.04331733286380768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,1,128,1,float16,float16,0,0.04186133543650309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,4,128,1,float16,float16,0,0.04603200157483419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,64,128,1,float16,float16,0,0.11995200316111247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,2,128,1,float16,float16,0,0.08575999736785889
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,64,128,1,float16,float16,0,0.06817600131034851
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,8,128,1,float16,float16,0,0.04608533283074697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,8,128,1,float16,float16,0,0.17317867279052734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,64,128,1,float16,float16,0,0.03298133363326391
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,1,128,1,float16,float16,0,0.025888000925381977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,64,128,1,float16,float16,0,0.02186666677395503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,4,128,1,float16,float16,0,0.019482667247454327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,4,128,1,float16,float16,0,0.028138667345046997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,2,128,1,float16,float16,0,0.019029332945744198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,8,128,1,float16,float16,0,0.027935999135176342
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,1,128,1,float16,float16,0,0.01882133384545644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,8,128,1,float16,float16,0,0.019391999890406925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,2,128,1,float16,float16,0,0.02613866577545802
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,1,128,1,float16,float16,0,0.014432000617186228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,64,128,1,float16,float16,0,0.015861333658297855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,2,128,1,float16,float16,0,0.01451733335852623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,8,128,1,float16,float16,0,0.014650666465361914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,4,128,1,float16,float16,0,0.014688000082969666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,4,128,1,float16,float16,0,0.09196799993515015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,64,128,1,float16,float16,0,0.013023999830087027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,1,128,1,float16,float16,0,0.011994666109482447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,2,128,1,float16,float16,0,0.011839999506870905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,4,128,1,float16,float16,0,0.012037333101034164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,8,128,1,float16,float16,0,0.011653333902359009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,1,128,1,float16,float16,0,0.2845226724942525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,2,128,1,float16,float16,0,0.29151999950408936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,4,128,1,float16,float16,0,0.3033813238143921
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,64,1,128,1,float16,float16,0,0.5606293280919393
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,64,8,128,1,float16,float16,0,0.601306676864624
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,64,128,1,float16,float16,0,0.3568426767985026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,1,128,1,float16,float16,0,0.14661866426467896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,8,128,1,float16,float16,0,0.3048693339029948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,64,4,128,1,float16,float16,0,0.6003733476003011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,64,128,1,float16,float16,0,0.18401066462198892
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,64,2,128,1,float16,float16,0,0.5754826863606771
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,64,128,1,float16,float16,0,0.09949333469072978
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,1,128,1,float16,float16,0,0.07307200133800507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,2,128,1,float16,float16,0,0.14820266763369241
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,2,128,1,float16,float16,0,0.07497600217660268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,4,128,1,float16,float16,0,0.07737066845099132
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,2,128,1,float16,float16,0,0.04104000081618627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,64,128,1,float16,float16,0,0.04859200119972229
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,4,128,1,float16,float16,0,0.15416000286738077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,8,128,1,float16,float16,0,0.07853866616884868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,1,128,1,float16,float16,0,0.040565334260463715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,8,128,1,float16,float16,0,0.1564533313115438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,64,128,1,float16,float16,0,0.027952000498771667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,4,128,1,float16,float16,0,0.025498665869235992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,4,128,1,float16,float16,0,0.04271999994913737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,8,128,1,float16,float16,0,0.0429066667954127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,1,128,1,float16,float16,0,0.02497066557407379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,2,128,1,float16,float16,0,0.025274666647116344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,8,128,1,float16,float16,0,0.02565866708755493
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,1,128,1,float16,float16,0,0.018405333161354065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,64,128,1,float16,float16,0,0.02014933278163274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,64,128,1,float16,float16,0,0.015487999965747198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,2,128,1,float16,float16,0,0.014058666924635569
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,2,128,1,float16,float16,0,0.018496000518401463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,8,128,1,float16,float16,0,0.018629333625237148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,1,128,1,float16,float16,0,0.014192000031471252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,4,128,1,float16,float16,0,0.018661333868900936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,8,128,1,float16,float16,0,0.014271999398867289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,64,128,1,float16,float16,0,0.012650666137536367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,4,128,1,float16,float16,0,0.01444799949725469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,2,128,1,float16,float16,0,0.011461333682139715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,4,128,1,float16,float16,0,0.011861333002646765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,1,128,1,float16,float16,0,0.011600000162919363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,8,128,1,float16,float16,0,0.011823999385039011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,0,0.01728533332546552
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,0,0.03443199892838796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,64,128,1,float16,float16,0,0.25224000215530396
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,0,0.024031999210516613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,0,0.04891733328501383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,0,0.09807999928792317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,0,0.06698666512966156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,64,128,1,float16,float16,0,0.1304480036099752
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,0,0.0406986673672994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,0,0.05670933425426483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,0,0.02499733368555705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,0,0.013781332721312841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,64,128,1,float16,float16,0,0.07005333403746287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,0,0.02938133229811986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,0,0.03602666656176249
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,0,0.020608000457286835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,0,0.027162666122118633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,0,0.01370666672786077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,0,0.022853332261244457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,0,0.013258667041858038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,64,128,1,float16,float16,0,0.04066666712363561
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,0,0.020053333292404812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,0,0.012794667234023413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,0,0.02257599929968516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,64,128,1,float16,float16,0,0.02476266771554947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,0,0.020128000527620316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,0,0.0194560003777345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,0,0.019802667200565338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,0,0.019808000574509304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,0,0.012719999998807907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,0,0.019071999937295914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,0,0.0189280000825723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,0,0.01933866615096728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,64,128,1,float16,float16,0,0.014522666732470194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,0,0.013013333082199097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,0,0.018309333672126133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,0,0.018351999421914417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,0,0.01834133391578992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,0,0.012805332740147909
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,0,0.012416000167528788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,64,128,1,float16,float16,0,0.018719999740521114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,0,0.012863999853531519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,64,128,1,float16,float16,0,0.011909333368142446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,0,0.012576000144084295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,1,128,1,float16,float16,0,72.50865681966145
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,4,128,1,float16,float16,0,73.35573323567708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,2,128,1,float16,float16,0,73.48593139648438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,48,128,1,float16,float16,0,75.45694986979167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,1,128,1,float16,float16,0,36.70391591389974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,48,128,1,float16,float16,0,37.53665669759115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,2,128,1,float16,float16,0,36.85007985432943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,48,128,1,float16,float16,0,19.034693400065105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,48,1,128,1,float16,float16,0,145.45995076497397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,48,2,128,1,float16,float16,0,147.16024780273438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,8,128,1,float16,float16,0,73.31158955891927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,48,4,128,1,float16,float16,0,149.02490234375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,4,128,1,float16,float16,0,37.051134745279946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,8,128,1,float16,float16,0,36.74391428629557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,48,8,128,1,float16,float16,0,150.29812622070312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,1,128,1,float16,float16,0,18.47752507527669
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,2,128,1,float16,float16,0,18.51346715291341
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,4,128,1,float16,float16,0,18.84876759847005
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,8,128,1,float16,float16,0,18.600500742594402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,48,128,1,float16,float16,0,43.133392333984375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,1,128,1,float16,float16,0,41.31376647949219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,2,128,1,float16,float16,0,41.72017161051432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,4,128,1,float16,float16,0,41.69031524658203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,48,128,1,float16,float16,0,21.281482696533203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,1,128,1,float16,float16,0,20.68604278564453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,48,1,128,1,float16,float16,0,83.83370971679688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,48,2,128,1,float16,float16,0,83.75015767415364
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,48,4,128,1,float16,float16,0,83.8398946126302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,48,8,128,1,float16,float16,0,85.50508626302083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,8,128,1,float16,float16,0,42.776275634765625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,48,128,1,float16,float16,0,10.90029271443685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,1,128,1,float16,float16,0,10.422304153442383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,2,128,1,float16,float16,0,10.45683733622233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,4,128,1,float16,float16,0,20.982256571451824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,4,128,1,float16,float16,0,10.661845525105795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,2,128,1,float16,float16,0,20.873099009195965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,8,128,1,float16,float16,0,20.930618286132812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,8,128,1,float16,float16,0,10.690123240152994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,1,128,1,float16,float16,0,29.082486470540363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,48,128,1,float16,float16,0,30.62482198079427
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,2,128,1,float16,float16,0,29.069605509440105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,4,128,1,float16,float16,0,29.51488494873047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,48,128,1,float16,float16,0,15.332138061523438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,1,128,1,float16,float16,0,14.47317377726237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,2,128,1,float16,float16,0,14.686468760172525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,48,1,128,1,float16,float16,0,58.65365091959635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,48,2,128,1,float16,float16,0,58.57038370768229
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,48,4,128,1,float16,float16,0,58.66667175292969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,48,8,128,1,float16,float16,0,58.36954243977865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,8,128,1,float16,float16,0,29.639012654622395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,48,128,1,float16,float16,0,7.599573135375977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,4,128,1,float16,float16,0,14.900800069173178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,8,128,1,float16,float16,0,14.604384104410807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,1,128,1,float16,float16,0,7.419797261555989
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,2,128,1,float16,float16,0,7.3567250569661455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,4,128,1,float16,float16,0,7.360789616902669
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,8,128,1,float16,float16,0,7.303610483805339
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,1,128,1,float16,float16,0,37.67956797281901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,2,128,1,float16,float16,0,37.51812235514323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,48,128,1,float16,float16,0,39.41694895426432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,4,128,1,float16,float16,0,37.946484883626304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,48,128,1,float16,float16,0,19.55344009399414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,2,128,1,float16,float16,0,18.679946899414062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,1,128,1,float16,float16,0,18.974266052246094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,48,128,1,float16,float16,0,9.889328002929688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,48,1,128,1,float16,float16,0,75.56184895833333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,48,2,128,1,float16,float16,0,76.18182881673177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,48,8,128,1,float16,float16,0,76.04175821940105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,8,128,1,float16,float16,0,38.359214782714844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,48,4,128,1,float16,float16,0,77.70793151855469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,4,128,1,float16,float16,0,18.926575978597004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,8,128,1,float16,float16,0,18.950171152750652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,48,128,1,float16,float16,0,4.887802759806315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,1,128,1,float16,float16,0,9.38491185506185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,1,128,1,float16,float16,0,4.692639986673991
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,2,128,1,float16,float16,0,9.389984130859375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,2,128,1,float16,float16,0,4.6903839111328125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,4,128,1,float16,float16,0,9.52231470743815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,4,128,1,float16,float16,0,4.702047983805339
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,8,128,1,float16,float16,0,9.423056284586588
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,8,128,1,float16,float16,0,4.735477447509766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,1,128,1,float16,float16,0,21.17126973470052
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,2,128,1,float16,float16,0,21.47595723470052
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,48,128,1,float16,float16,0,22.980662027994793
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,4,128,1,float16,float16,0,21.92736562093099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,48,128,1,float16,float16,0,11.324053446451822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,1,128,1,float16,float16,0,10.671669006347656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,2,128,1,float16,float16,0,10.730058034261068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,48,1,128,1,float16,float16,0,43.24034627278646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,48,2,128,1,float16,float16,0,43.61470031738281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,48,4,128,1,float16,float16,0,43.73951212565104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,48,128,1,float16,float16,0,5.59988276163737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,48,8,128,1,float16,float16,0,44.03461201985677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,8,128,1,float16,float16,0,21.85186258951823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,4,128,1,float16,float16,0,10.895258585611979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,48,128,1,float16,float16,0,2.8249546686808267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,1,128,1,float16,float16,0,5.25706672668457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,2,128,1,float16,float16,0,5.3606611887613935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,8,128,1,float16,float16,0,11.000410715738932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,1,128,1,float16,float16,0,2.681781450907389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,8,128,1,float16,float16,0,5.39085324605306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,4,128,1,float16,float16,0,5.2931413650512695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,2,128,1,float16,float16,0,2.6881707509358725
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,4,128,1,float16,float16,0,2.659269332885742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,8,128,1,float16,float16,0,2.754122734069824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,1,128,1,float16,float16,0,19.497477213541668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,2,128,1,float16,float16,0,19.91376495361328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,4,128,1,float16,float16,0,20.109451293945312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,48,128,1,float16,float16,0,21.420399983723957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,1,128,1,float16,float16,0,9.826645533243815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,48,128,1,float16,float16,0,10.653002421061197
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,2,128,1,float16,float16,0,9.82644780476888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,48,2,128,1,float16,float16,0,40.06543477376302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,48,1,128,1,float16,float16,0,39.99615987141927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,48,4,128,1,float16,float16,0,40.16763814290365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,48,8,128,1,float16,float16,0,40.78245290120443
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,48,128,1,float16,float16,0,5.184213320414226
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,8,128,1,float16,float16,0,20.11950937906901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,4,128,1,float16,float16,0,10.120762507120768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,8,128,1,float16,float16,0,10.124165217081705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,4,128,1,float16,float16,0,4.876864115397136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,1,128,1,float16,float16,0,4.807557423909505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,2,128,1,float16,float16,0,4.811722755432129
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,48,128,1,float16,float16,0,2.6257759730021157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,1,128,1,float16,float16,0,2.419461409250895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,8,128,1,float16,float16,0,4.955914815266927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,2,128,1,float16,float16,0,2.474266688028971
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,4,128,1,float16,float16,0,2.471754709879557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,48,128,1,float16,float16,0,1.4084266026814778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,2,128,1,float16,float16,0,1.2534720102945964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,1,128,1,float16,float16,0,1.2183039983113606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,8,128,1,float16,float16,0,2.4708479245503745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,4,128,1,float16,float16,0,1.3105440139770508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,8,128,1,float16,float16,0,1.291434685389201
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,1,128,1,float16,float16,0,11.38213857014974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,2,128,1,float16,float16,0,11.607599894205729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,48,128,1,float16,float16,0,12.60263442993164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,1,128,1,float16,float16,0,5.380053202311198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,48,1,128,1,float16,float16,0,22.64026133219401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,48,2,128,1,float16,float16,0,23.47667694091797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,48,128,1,float16,float16,0,6.1654612223307295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,48,4,128,1,float16,float16,0,23.413599650065105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,8,128,1,float16,float16,0,11.868512471516928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,4,128,1,float16,float16,0,11.7118771870931
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,48,8,128,1,float16,float16,0,23.726341247558594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,2,128,1,float16,float16,0,5.521685282389323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,1,128,1,float16,float16,0,2.718031883239746
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,48,128,1,float16,float16,0,3.0919361114501953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,2,128,1,float16,float16,0,2.86517333984375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,4,128,1,float16,float16,0,2.8560587565104165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,4,128,1,float16,float16,0,5.668277104695638
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,48,128,1,float16,float16,0,1.5690186818440754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,8,128,1,float16,float16,0,5.866448084513347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,1,128,1,float16,float16,0,1.3762720425923665
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,48,128,1,float16,float16,0,0.8345119953155518
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,1,128,1,float16,float16,0,0.7203893661499023
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,8,128,1,float16,float16,0,2.8295841217041016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,4,128,1,float16,float16,0,1.4589866002400715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,2,128,1,float16,float16,0,0.7803359826405843
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,2,128,1,float16,float16,0,1.41483736038208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,8,128,1,float16,float16,0,1.4271680514017742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,4,128,1,float16,float16,0,0.7527360121409098
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,8,128,1,float16,float16,0,0.769050677617391
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,1,128,1,float16,float16,0,10.708688100179037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,2,128,1,float16,float16,0,10.70040512084961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,4,128,1,float16,float16,0,11.287439982096354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,48,128,1,float16,float16,0,12.490117390950521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,1,128,1,float16,float16,0,5.155695915222168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,48,128,1,float16,float16,0,6.00437863667806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,2,128,1,float16,float16,0,5.160863876342773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,48,1,128,1,float16,float16,0,21.14134343465169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,48,4,128,1,float16,float16,0,22.665674845377605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,48,8,128,1,float16,float16,0,23.030601501464844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,8,128,1,float16,float16,0,11.281285603841146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,4,128,1,float16,float16,0,5.4010880788167315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,48,128,1,float16,float16,0,3.0329386393229165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,1,128,1,float16,float16,0,2.5664265950520835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,8,128,1,float16,float16,0,5.436469395955403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,2,128,1,float16,float16,0,2.623413403828939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,4,128,1,float16,float16,0,2.732288042704264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,48,128,1,float16,float16,0,1.525536060333252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,48,2,128,1,float16,float16,0,21.805081685384113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,8,128,1,float16,float16,0,2.7378292083740234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,2,128,1,float16,float16,0,1.305402676264445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,48,128,1,float16,float16,0,0.805893341700236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,4,128,1,float16,float16,0,1.3816693623860676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,1,128,1,float16,float16,0,1.271135965983073
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,8,128,1,float16,float16,0,1.3708426157633464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,1,128,1,float16,float16,0,0.6617493232091268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,4,128,1,float16,float16,0,0.7018559773763021
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,48,128,1,float16,float16,0,0.4152640104293823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,1,128,1,float16,float16,0,0.35496532917022705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,4,128,1,float16,float16,0,0.3801493247350057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,2,128,1,float16,float16,0,0.35549334685007733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,8,128,1,float16,float16,0,0.6938026746114095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,8,128,1,float16,float16,0,0.37225600083669025
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,2,128,1,float16,float16,0,0.6728746891021729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,1,128,1,float16,float16,0,6.17909304300944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,2,128,1,float16,float16,0,6.38644282023112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,48,128,1,float16,float16,0,7.577615737915039
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,4,128,1,float16,float16,0,6.237781524658203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,1,128,1,float16,float16,0,2.9641545613606772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,48,4,128,1,float16,float16,0,13.436021169026693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,48,8,128,1,float16,float16,0,13.658677419026693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,48,128,1,float16,float16,0,3.660208066304525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,8,128,1,float16,float16,0,6.446661631266276
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,48,1,128,1,float16,float16,0,12.551413218180338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,2,128,1,float16,float16,0,3.0881760915120444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,48,2,128,1,float16,float16,0,13.024122873942057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,1,128,1,float16,float16,0,1.4881439208984375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,48,128,1,float16,float16,0,1.877247969309489
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,8,128,1,float16,float16,0,3.220831871032715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,4,128,1,float16,float16,0,3.154229482014974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,2,128,1,float16,float16,0,1.5528373718261719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,48,128,1,float16,float16,0,0.9294453461964926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,2,128,1,float16,float16,0,0.790783961613973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,4,128,1,float16,float16,0,0.8354933261871338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,48,128,1,float16,float16,0,0.48817066351572674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,1,128,1,float16,float16,0,0.39668798446655273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,8,128,1,float16,float16,0,0.8258026440938314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,4,128,1,float16,float16,0,1.6182079315185547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,8,128,1,float16,float16,0,1.6102666854858398
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,1,128,1,float16,float16,0,0.7997600237528483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,2,128,1,float16,float16,0,0.4113653500874837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,4,128,1,float16,float16,0,0.4297279914220174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,48,128,1,float16,float16,0,0.2648800015449524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,8,128,1,float16,float16,0,0.4246986707051595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,2,128,1,float16,float16,0,0.22401599089304605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,1,128,1,float16,float16,0,0.22025599082310995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,4,128,1,float16,float16,0,0.24791999657948813
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,8,128,1,float16,float16,0,0.23494933048884073
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,1,128,1,float16,float16,0,6.091136296590169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,48,128,1,float16,float16,0,7.913951873779297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,2,128,1,float16,float16,0,5.859039942423503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,48,1,128,1,float16,float16,0,12.257439931233725
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,48,8,128,1,float16,float16,0,13.594987233479818
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,48,2,128,1,float16,float16,0,12.732111612955729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,48,128,1,float16,float16,0,3.8281758626302085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,1,128,1,float16,float16,0,2.9263572692871094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,8,128,1,float16,float16,0,6.585770924886067
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,4,128,1,float16,float16,0,3.194239934285482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,48,4,128,1,float16,float16,0,12.869781494140625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,4,128,1,float16,float16,0,6.663295745849609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,1,128,1,float16,float16,0,1.429093360900879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,48,128,1,float16,float16,0,1.8977012634277344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,4,128,1,float16,float16,0,1.5812320709228516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,2,128,1,float16,float16,0,1.492784023284912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,8,128,1,float16,float16,0,3.1941067377726235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,1,128,1,float16,float16,0,0.730725367863973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,2,128,1,float16,float16,0,2.908869425455729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,2,128,1,float16,float16,0,0.7599946657816569
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,1,128,1,float16,float16,0,0.3808319965998332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,48,128,1,float16,float16,0,0.49612267812093097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,4,128,1,float16,float16,0,0.8056533336639404
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,48,128,1,float16,float16,0,0.9693813323974609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,8,128,1,float16,float16,0,0.8150826295216879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,48,128,1,float16,float16,0,0.26442132393519086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,4,128,1,float16,float16,0,0.41678400834401447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,2,128,1,float16,float16,0,0.2098133365313212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,8,128,1,float16,float16,0,0.4251946608225505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,1,128,1,float16,float16,0,0.2025973399480184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,4,128,1,float16,float16,0,0.22473067045211792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,8,128,1,float16,float16,0,0.2255093256632487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,1,128,1,float16,float16,0,0.11502933502197266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,2,128,1,float16,float16,0,0.400383989016215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,2,128,1,float16,float16,0,0.11774933338165283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,4,128,1,float16,float16,0,0.12213333447774251
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,48,128,1,float16,float16,0,0.1476959983507792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,8,128,1,float16,float16,0,0.12681066989898682
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,8,128,1,float16,float16,0,1.593989372253418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,1,128,1,float16,float16,0,3.622346560160319
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,48,128,1,float16,float16,0,5.538554509480794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,48,1,128,1,float16,float16,0,7.72708257039388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,2,128,1,float16,float16,0,3.8603572845458984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,4,128,1,float16,float16,0,4.239472071329753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,48,128,1,float16,float16,0,2.7662026087443032
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,1,128,1,float16,float16,0,1.8654932975769043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,2,128,1,float16,float16,0,1.9546826680501301
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,4,128,1,float16,float16,0,2.1369172732035318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,48,2,128,1,float16,float16,0,8.074991861979166
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,48,128,1,float16,float16,0,1.3782399495442708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,1,128,1,float16,float16,0,0.9243626594543457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,8,128,1,float16,float16,0,2.1624372800191245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,2,128,1,float16,float16,0,0.970245361328125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,48,128,1,float16,float16,0,0.705509344736735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,8,128,1,float16,float16,0,1.0849599838256836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,4,128,1,float16,float16,0,1.1289599736531575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,1,128,1,float16,float16,0,0.4633920192718506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,2,128,1,float16,float16,0,0.49515199661254883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,48,4,128,1,float16,float16,0,8.929424285888672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,1,128,1,float16,float16,0,0.24256000916163126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,2,128,1,float16,float16,0,0.25860265890757245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,4,128,1,float16,float16,0,0.5467466513315836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,48,128,1,float16,float16,0,0.367301344871521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,8,128,1,float16,float16,0,0.5514613389968872
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,48,8,128,1,float16,float16,0,8.996112187703451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,48,128,1,float16,float16,0,0.19101866086324057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,1,128,1,float16,float16,0,0.1348960002263387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,8,128,1,float16,float16,0,0.2852960030237834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,2,128,1,float16,float16,0,0.13763733704884848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,48,128,1,float16,float16,0,0.10558399558067322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,4,128,1,float16,float16,0,0.15176533659299216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,1,128,1,float16,float16,0,0.07580266892910004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,8,128,1,float16,float16,0,0.08514133095741272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,4,128,1,float16,float16,0,0.08509332935015361
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,2,128,1,float16,float16,0,0.0775733341773351
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,1,128,1,float16,float16,0,0.047093331813812256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,48,128,1,float16,float16,0,0.06408533453941345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,8,128,1,float16,float16,0,4.33513609568278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,2,128,1,float16,float16,0,0.04737600187460581
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,4,128,1,float16,float16,0,0.05073066552480062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,8,128,1,float16,float16,0,0.04933866858482361
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,4,128,1,float16,float16,0,0.28275199731191
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,1,128,1,float16,float16,0,1.302575985590617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,8,128,1,float16,float16,0,0.15610133608182272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,48,128,1,float16,float16,0,2.2659680048624673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,48,1,128,1,float16,float16,0,2.644181410471598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,2,128,1,float16,float16,0,1.3941866556803386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,48,2,128,1,float16,float16,0,2.807546615600586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,48,128,1,float16,float16,0,1.138698657353719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,48,4,128,1,float16,float16,0,3.2681172688802085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,2,128,1,float16,float16,0,0.6909013589223226
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,1,128,1,float16,float16,0,0.6296159823735555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,8,128,1,float16,float16,0,1.6332586606343586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,4,128,1,float16,float16,0,1.6034026145935059
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,4,128,1,float16,float16,0,0.8165120283762614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,48,128,1,float16,float16,0,0.579530676205953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,1,128,1,float16,float16,0,0.3239786624908447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,48,8,128,1,float16,float16,0,3.247983932495117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,8,128,1,float16,float16,0,0.4198240041732788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,1,128,1,float16,float16,0,0.17099199692408243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,48,128,1,float16,float16,0,0.30001600583394367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,4,128,1,float16,float16,0,0.4046880006790161
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,8,128,1,float16,float16,0,0.8087200323740641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,4,128,1,float16,float16,0,0.2109866738319397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,48,128,1,float16,float16,0,0.15729066729545593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,2,128,1,float16,float16,0,0.3540586630503337
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,8,128,1,float16,float16,0,0.21386667092641196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,1,128,1,float16,float16,0,0.09429333607355754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,4,128,1,float16,float16,0,0.11401066184043884
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,48,128,1,float16,float16,0,0.08586133519808452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,4,128,1,float16,float16,0,0.06200533111890157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,2,128,1,float16,float16,0,0.05539200206597646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,8,128,1,float16,float16,0,0.11468266447385152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,1,128,1,float16,float16,0,0.05349333087603251
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,8,128,1,float16,float16,0,0.06493333478768666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,2,128,1,float16,float16,0,0.10009599725405376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,48,128,1,float16,float16,0,0.04930666585763296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,1,128,1,float16,float16,0,0.03213333338499069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,2,128,1,float16,float16,0,0.03297066688537598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,8,128,1,float16,float16,0,0.03527999917666117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,4,128,1,float16,float16,0,0.03440000116825104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,2,128,1,float16,float16,0,0.18363199631373087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,48,128,1,float16,float16,0,0.026261332134405773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,1,128,1,float16,float16,0,0.022154666483402252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,2,128,1,float16,float16,0,0.02160000056028366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,8,128,1,float16,float16,0,0.02330133318901062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,4,128,1,float16,float16,0,0.023141334454218548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,1,128,1,float16,float16,0,0.4917173385620117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,48,128,1,float16,float16,0,1.1034133434295654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,2,128,1,float16,float16,0,0.5493119955062866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,48,1,128,1,float16,float16,0,1.1588373184204102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,48,2,128,1,float16,float16,0,1.2435893217722576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,4,128,1,float16,float16,0,0.6580746571222941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,48,128,1,float16,float16,0,0.5507093270619711
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,48,8,128,1,float16,float16,0,1.5069653193155925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,48,4,128,1,float16,float16,0,1.5114293098449707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,1,128,1,float16,float16,0,0.24829334020614624
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,2,128,1,float16,float16,0,0.27721599737803143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,2,128,1,float16,float16,0,0.14461867014567056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,4,128,1,float16,float16,0,0.33161600430806476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,8,128,1,float16,float16,0,0.6762666702270508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,8,128,1,float16,float16,0,0.17483200629552206
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,48,128,1,float16,float16,0,0.1463573376337687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,1,128,1,float16,float16,0,0.07410133381684621
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,8,128,1,float16,float16,0,0.3424053192138672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,1,128,1,float16,float16,0,0.13301866253217062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,4,128,1,float16,float16,0,0.09269866347312927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,4,128,1,float16,float16,0,0.1715679963429769
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,2,128,1,float16,float16,0,0.07992533346017201
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,48,128,1,float16,float16,0,0.07961600025494893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,8,128,1,float16,float16,0,0.09379733602205913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,48,128,1,float16,float16,0,0.27828800678253174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,2,128,1,float16,float16,0,0.04496533175309499
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,1,128,1,float16,float16,0,0.041573333243529
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,4,128,1,float16,float16,0,0.049226666490236916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,1,128,1,float16,float16,0,0.024645333488782246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,8,128,1,float16,float16,0,0.05472533404827118
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,2,128,1,float16,float16,0,0.01757866640885671
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,4,128,1,float16,float16,0,0.027893332143624622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,48,128,1,float16,float16,0,0.04397333165009817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,1,128,1,float16,float16,0,0.01661866654952367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,48,128,1,float16,float16,0,0.020981334149837494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,8,128,1,float16,float16,0,0.02824000020821889
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,2,128,1,float16,float16,0,0.02603200078010559
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,4,128,1,float16,float16,0,0.018079999834299088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,48,128,1,float16,float16,0,0.015295999745527903
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,8,128,1,float16,float16,0,0.018437333405017853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,2,128,1,float16,float16,0,0.013253333667914072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,1,128,1,float16,float16,0,0.013733333597580591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,4,128,1,float16,float16,0,0.014085333794355392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,8,128,1,float16,float16,0,0.013631999492645264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,1,128,1,float16,float16,0,0.24542399247487387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,2,128,1,float16,float16,0,0.27292799949645996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,4,128,1,float16,float16,0,0.3272639910380046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,8,128,1,float16,float16,0,0.33614933490753174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,48,128,1,float16,float16,0,0.5347573359807333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,48,4,128,1,float16,float16,0,0.6396746635437012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,48,8,128,1,float16,float16,0,0.6571199893951416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,1,128,1,float16,float16,0,0.13314132889111838
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,1,128,1,float16,float16,0,0.0747680018345515
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,48,1,128,1,float16,float16,0,0.475381334622701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,2,128,1,float16,float16,0,0.14654399951299033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,48,128,1,float16,float16,0,0.28045332431793213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,48,2,128,1,float16,float16,0,0.5324053366978964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,2,128,1,float16,float16,0,0.07991999884446462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,48,128,1,float16,float16,0,0.1458080013593038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,2,128,1,float16,float16,0,0.04432533184687296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,8,128,1,float16,float16,0,0.09511466821034749
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,8,128,1,float16,float16,0,0.054714664816856384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,8,128,1,float16,float16,0,0.17433067162831625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,48,128,1,float16,float16,0,0.04399466514587402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,1,128,1,float16,float16,0,0.024085332949956257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,4,128,1,float16,float16,0,0.05003733436266581
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,4,128,1,float16,float16,0,0.027301333844661713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,48,128,1,float16,float16,0,0.07995200157165527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,8,128,1,float16,float16,0,0.027829334139823914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,48,128,1,float16,float16,0,0.020234666764736176
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,2,128,1,float16,float16,0,0.016528000434239704
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,2,128,1,float16,float16,0,0.02534399926662445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,4,128,1,float16,float16,0,0.017973333597183228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,48,128,1,float16,float16,0,0.01526933287580808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,8,128,1,float16,float16,0,0.0183999997874101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,1,128,1,float16,float16,0,0.016117333124081295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,1,128,1,float16,float16,0,0.012917333592971167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,2,128,1,float16,float16,0,0.013007999708255133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,4,128,1,float16,float16,0,0.013450667262077332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,8,128,1,float16,float16,0,0.013343999783198038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,4,128,1,float16,float16,0,0.09196266531944275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,4,128,1,float16,float16,0,0.16986133654912314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,2,128,1,float16,float16,0,0.012128000458081564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,1,128,1,float16,float16,0,0.04060266663630804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,48,128,1,float16,float16,0,0.013242666920026144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,4,128,1,float16,float16,0,0.012298667182525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,8,128,1,float16,float16,0,0.012346666306257248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,1,128,1,float16,float16,0,0.229802668094635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,2,128,1,float16,float16,0,0.2398293415705363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,48,1,128,1,float16,float16,0,0.4452213446299235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,48,2,128,1,float16,float16,0,0.4686559836069743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,48,4,128,1,float16,float16,0,0.5170453389485677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,48,128,1,float16,float16,0,0.17755732933680216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,48,8,128,1,float16,float16,0,0.5177706480026245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,4,128,1,float16,float16,0,0.26365866263707477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,8,128,1,float16,float16,0,0.2653333346048991
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,48,128,1,float16,float16,0,0.34482133388519287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,48,128,1,float16,float16,0,0.096261332432429
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,4,128,1,float16,float16,0,0.1390880048274994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,1,128,1,float16,float16,0,0.12277332941691081
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,48,128,1,float16,float16,0,0.05299200117588043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,2,128,1,float16,float16,0,0.1276800036430359
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,8,128,1,float16,float16,0,0.0776693324247996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,1,128,1,float16,float16,0,0.06443733473618825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,8,128,1,float16,float16,0,0.1402239998181661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,4,128,1,float16,float16,0,0.07467199862003326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,1,128,1,float16,float16,0,0.03489066660404205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,2,128,1,float16,float16,0,0.03623999903599421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,48,128,1,float16,float16,0,0.026047999660174053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,8,128,1,float16,float16,0,0.03950933367013931
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,4,128,1,float16,float16,0,0.038431999584039055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,4,128,1,float16,float16,0,0.023717333873112995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,2,128,1,float16,float16,0,0.02179733415444692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,1,128,1,float16,float16,0,0.021349333226680756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,8,128,1,float16,float16,0,0.02369066576162974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,48,128,1,float16,float16,0,0.01669866715868314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,2,128,1,float16,float16,0,0.067071999112765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,2,128,1,float16,float16,0,0.014901333798964819
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,1,128,1,float16,float16,0,0.015103999525308609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,4,128,1,float16,float16,0,0.01524266724785169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,1,128,1,float16,float16,0,0.012074666718641916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,1,128,1,float16,float16,0,0.011674666156371435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,8,128,1,float16,float16,0,0.015461333096027374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,2,128,1,float16,float16,0,0.011936000237862269
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,8,128,1,float16,float16,0,0.0122079998254776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,48,128,1,float16,float16,0,0.013141332815090815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,4,128,1,float16,float16,0,0.012373333175977072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,48,128,1,float16,float16,0,0.012821332861979803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,2,128,1,float16,float16,0,0.011621333658695221
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,4,128,1,float16,float16,0,0.011781333635250727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,8,128,1,float16,float16,0,0.011882666498422623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,1,128,1,float16,float16,0,0.2193173368771871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,2,128,1,float16,float16,0,0.22539732853571573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,4,128,1,float16,float16,0,0.23786133527755737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,48,2,128,1,float16,float16,0,0.44412267208099365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,48,128,1,float16,float16,0,0.1427893340587616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,8,128,1,float16,float16,0,0.23863466580708823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,48,1,128,1,float16,float16,0,0.4294133186340332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,48,128,1,float16,float16,0,0.27534933884938556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,1,128,1,float16,float16,0,0.1134986678759257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,48,4,128,1,float16,float16,0,0.4656746784845988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,48,8,128,1,float16,float16,0,0.4691786766052246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,2,128,1,float16,float16,0,0.1139359970887502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,4,128,1,float16,float16,0,0.1204853355884552
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,2,128,1,float16,float16,0,0.059205333391825356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,8,128,1,float16,float16,0,0.12342400352160136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,8,128,1,float16,float16,0,0.06217066446940104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,1,128,1,float16,float16,0,0.0339626669883728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,48,128,1,float16,float16,0,0.07749333480993907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,2,128,1,float16,float16,0,0.033946665624777474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,48,128,1,float16,float16,0,0.023567999402681988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,8,128,1,float16,float16,0,0.03626133253177007
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,4,128,1,float16,float16,0,0.06158400078614553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,4,128,1,float16,float16,0,0.035887998839219414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,1,128,1,float16,float16,0,0.0580266664425532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,4,128,1,float16,float16,0,0.021520001192887623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,8,128,1,float16,float16,0,0.021583999196688335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,48,128,1,float16,float16,0,0.03895466774702072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,2,128,1,float16,float16,0,0.020960000654061634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,1,128,1,float16,float16,0,0.014442666123310724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,48,128,1,float16,float16,0,0.015530666957298914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,1,128,1,float16,float16,0,0.021055998901526134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,2,128,1,float16,float16,0,0.014666666587193808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,4,128,1,float16,float16,0,0.014565333724021912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,8,128,1,float16,float16,0,0.01462399959564209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,1,128,1,float16,float16,0,0.011792000383138657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,2,128,1,float16,float16,0,0.011823999385039011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,4,128,1,float16,float16,0,0.01198400060335795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,48,128,1,float16,float16,0,0.013248000293970108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,8,128,1,float16,float16,0,0.011957333733638128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,48,128,1,float16,float16,0,0.012906666845083237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,1,128,1,float16,float16,0,0.011418666690587997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,4,128,1,float16,float16,0,0.011621333658695221
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,8,128,1,float16,float16,0,0.011744000017642975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,2,128,1,float16,float16,0,0.011525332927703857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,0,0.016666666915019352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,48,128,1,float16,float16,0,0.19082132975260416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,0,0.031530665854612984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,0,0.04454400142033895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,0,0.022704000274340313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,0,0.09285333752632141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,0,0.061978667974472046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,48,128,1,float16,float16,0,0.10046399633089702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,0,0.03802666564782461
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,0,0.053504000107447304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,0,0.013781332721312841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,0,0.028223998844623566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,0,0.023647998770078022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,0,0.034789333740870156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,48,128,1,float16,float16,0,0.056090667843818665
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,0,0.020074666788180668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,0,0.01339200014869372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,0,0.023183998962243397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,48,128,1,float16,float16,0,0.03402666747570038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,0,0.02735466758410136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,0,0.01332266628742218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,0,0.01958400011062622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,0,0.020021333048741024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,0,0.02237333357334137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,0,0.012863999853531519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,0,0.019461333751678467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,0,0.01937066639463107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,48,128,1,float16,float16,0,0.014618666221698126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,48,128,1,float16,float16,0,0.020997333029905956
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,0,0.019424000134070713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,0,0.012682666381200155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,0,0.01883200059334437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,0,0.01889066646496455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,48,128,1,float16,float16,0,0.01190399999419848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,0,0.019445333629846573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,0,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,0,0.018474667022625606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,0,0.01848000039656957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,0,0.018511999398469925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,0,0.012448000411192576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,0,0.012426666915416718
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,48,128,1,float16,float16,0,0.01202133297920227
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,0,0.012378666549921036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,0,0.0124746672809124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,1,128,1,float16,float16,0,60.99896240234375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,2,128,1,float16,float16,0,61.2334950764974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,40,128,1,float16,float16,0,62.8989003499349
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,4,128,1,float16,float16,0,61.93732706705729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,40,128,1,float16,float16,0,31.697919209798176
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,1,128,1,float16,float16,0,30.771647135416668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,2,128,1,float16,float16,0,30.80742899576823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,40,2,128,1,float16,float16,0,122.84543863932292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,40,128,1,float16,float16,0,15.915679931640625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,40,4,128,1,float16,float16,0,122.91036987304688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,40,8,128,1,float16,float16,0,122.7838643391927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,40,1,128,1,float16,float16,0,124.41017659505208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,8,128,1,float16,float16,0,61.92505900065104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,4,128,1,float16,float16,0,31.011871337890625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,8,128,1,float16,float16,0,31.122848510742188
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,1,128,1,float16,float16,0,15.270058949788412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,2,128,1,float16,float16,0,15.528949737548828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,4,128,1,float16,float16,0,15.41494878133138
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,8,128,1,float16,float16,0,15.44927978515625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,2,128,1,float16,float16,0,34.87669372558594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,1,128,1,float16,float16,0,34.85454813639323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,4,128,1,float16,float16,0,35.069358825683594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,40,128,1,float16,float16,0,35.960400899251304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,1,128,1,float16,float16,0,17.276570638020832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,2,128,1,float16,float16,0,17.329818725585938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,40,128,1,float16,float16,0,18.099520365397137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,40,1,128,1,float16,float16,0,70.74975077311198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,40,2,128,1,float16,float16,0,69.70260620117188
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,40,4,128,1,float16,float16,0,70.45237223307292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,40,8,128,1,float16,float16,0,70.97824605305989
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,40,128,1,float16,float16,0,9.067509333292643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,1,128,1,float16,float16,0,8.764298756917318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,2,128,1,float16,float16,0,8.821439743041992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,8,128,1,float16,float16,0,34.999621073404946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,4,128,1,float16,float16,0,8.824864069620768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,4,128,1,float16,float16,0,17.4998296101888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,8,128,1,float16,float16,0,17.49523671468099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,8,128,1,float16,float16,0,8.911120096842447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,1,128,1,float16,float16,0,24.12298075358073
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,2,128,1,float16,float16,0,24.223673502604168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,40,128,1,float16,float16,0,25.624277750651043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,4,128,1,float16,float16,0,24.703631083170574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,40,128,1,float16,float16,0,12.811925252278646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,2,128,1,float16,float16,0,12.255050659179688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,1,128,1,float16,float16,0,12.098709106445312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,40,1,128,1,float16,float16,0,48.79200744628906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,40,2,128,1,float16,float16,0,49.61982218424479
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,40,4,128,1,float16,float16,0,49.17459615071615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,40,8,128,1,float16,float16,0,48.97481791178385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,40,128,1,float16,float16,0,6.329989115397136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,8,128,1,float16,float16,0,24.47534434000651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,1,128,1,float16,float16,0,6.047711690266927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,4,128,1,float16,float16,0,12.254175821940104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,8,128,1,float16,float16,0,12.3647092183431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,2,128,1,float16,float16,0,6.07646369934082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,4,128,1,float16,float16,0,6.1494185129801435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,8,128,1,float16,float16,0,6.161002477010091
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,1,128,1,float16,float16,0,31.232805887858074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,40,128,1,float16,float16,0,33.279136657714844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,2,128,1,float16,float16,0,31.741221110026043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,4,128,1,float16,float16,0,31.967140197753906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,1,128,1,float16,float16,0,15.700703938802084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,40,128,1,float16,float16,0,16.56333287556966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,2,128,1,float16,float16,0,15.885370890299479
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,40,1,128,1,float16,float16,0,62.46098327636719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,40,4,128,1,float16,float16,0,63.94407653808594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,40,2,128,1,float16,float16,0,63.94469197591146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,40,8,128,1,float16,float16,0,64.17519632975261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,8,128,1,float16,float16,0,31.757647196451824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,40,128,1,float16,float16,0,8.191978454589844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,1,128,1,float16,float16,0,7.74995231628418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,4,128,1,float16,float16,0,16.310340881347656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,8,128,1,float16,float16,0,16.038314819335938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,40,128,1,float16,float16,0,4.0686079661051435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,1,128,1,float16,float16,0,3.819845199584961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,2,128,1,float16,float16,0,7.814842859903972
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,2,128,1,float16,float16,0,3.8972479502360025
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,4,128,1,float16,float16,0,8.007274627685547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,8,128,1,float16,float16,0,3.916927973429362
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,4,128,1,float16,float16,0,3.9614718755086265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,8,128,1,float16,float16,0,7.856880187988281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,1,128,1,float16,float16,0,17.86795171101888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,2,128,1,float16,float16,0,18.11956787109375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,40,128,1,float16,float16,0,19.28375498453776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,4,128,1,float16,float16,0,18.383071899414062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,40,128,1,float16,float16,0,9.663455963134766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,1,128,1,float16,float16,0,8.981952031453451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,2,128,1,float16,float16,0,8.977888107299805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,40,1,128,1,float16,float16,0,35.70032501220703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,40,2,128,1,float16,float16,0,36.45911407470703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,40,4,128,1,float16,float16,0,36.70819854736328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,40,128,1,float16,float16,0,4.65774408976237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,40,8,128,1,float16,float16,0,36.39701843261719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,8,128,1,float16,float16,0,18.40003712972005
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,1,128,1,float16,float16,0,4.379232088724772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,40,128,1,float16,float16,0,2.3675146102905273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,4,128,1,float16,float16,0,9.076255798339844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,8,128,1,float16,float16,0,9.20852788289388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,1,128,1,float16,float16,0,2.2459306716918945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,2,128,1,float16,float16,0,2.238405386606852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,4,128,1,float16,float16,0,4.4364213943481445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,8,128,1,float16,float16,0,4.499375979105632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,2,128,1,float16,float16,0,4.387845357259114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,4,128,1,float16,float16,0,2.233344078063965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,8,128,1,float16,float16,0,2.3051253954569497
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,2,128,1,float16,float16,0,16.303775787353516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,1,128,1,float16,float16,0,16.448986053466797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,40,128,1,float16,float16,0,17.927594502766926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,4,128,1,float16,float16,0,17.03551483154297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,2,128,1,float16,float16,0,8.183594385782877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,1,128,1,float16,float16,0,8.00596809387207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,40,128,1,float16,float16,0,8.838544209798178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,40,2,128,1,float16,float16,0,33.24998982747396
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,40,4,128,1,float16,float16,0,33.73875172932943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,40,8,128,1,float16,float16,0,33.95544942220052
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,40,1,128,1,float16,float16,0,33.44586690266927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,8,128,1,float16,float16,0,16.978490193684895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,40,128,1,float16,float16,0,4.348031997680664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,1,128,1,float16,float16,0,4.0512746175130205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,40,128,1,float16,float16,0,2.2176586786905923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,4,128,1,float16,float16,0,8.335413614908854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,2,128,1,float16,float16,0,4.018575986226399
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,8,128,1,float16,float16,0,8.337258656819662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,4,128,1,float16,float16,0,4.140218734741211
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,1,128,1,float16,float16,0,1.952181339263916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,40,128,1,float16,float16,0,1.1620319684346516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,4,128,1,float16,float16,0,2.0436107317606607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,2,128,1,float16,float16,0,2.0271199544270835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,8,128,1,float16,float16,0,2.112858613332113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,2,128,1,float16,float16,0,1.055957317352295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,1,128,1,float16,float16,0,1.059605360031128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,8,128,1,float16,float16,0,4.008048057556152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,4,128,1,float16,float16,0,1.0502080122629802
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,8,128,1,float16,float16,0,1.075002670288086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,1,128,1,float16,float16,0,9.574064254760742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,2,128,1,float16,float16,0,9.696191787719727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,4,128,1,float16,float16,0,9.718645095825195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,40,128,1,float16,float16,0,10.562565485636393
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,1,128,1,float16,float16,0,4.561349232991536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,40,128,1,float16,float16,0,5.186613400777181
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,40,1,128,1,float16,float16,0,18.707317352294922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,40,2,128,1,float16,float16,0,19.178869883219402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,2,128,1,float16,float16,0,4.513327916463216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,40,4,128,1,float16,float16,0,19.95474624633789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,40,8,128,1,float16,float16,0,19.768783569335938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,8,128,1,float16,float16,0,9.989456176757812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,4,128,1,float16,float16,0,4.815013249715169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,1,128,1,float16,float16,0,2.3082027435302734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,8,128,1,float16,float16,0,4.8165334065755205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,40,128,1,float16,float16,0,2.560943921407064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,2,128,1,float16,float16,0,2.323333263397217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,4,128,1,float16,float16,0,2.433274745941162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,40,128,1,float16,float16,0,1.33350404103597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,1,128,1,float16,float16,0,1.1994826793670654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,8,128,1,float16,float16,0,2.460757255554199
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,40,128,1,float16,float16,0,0.6985226472218832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,2,128,1,float16,float16,0,0.6227680047353109
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,1,128,1,float16,float16,0,0.6028853257497152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,4,128,1,float16,float16,0,1.2098453044891357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,2,128,1,float16,float16,0,1.168826659520467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,4,128,1,float16,float16,0,0.6466399828592936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,8,128,1,float16,float16,0,1.2040639718373616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,8,128,1,float16,float16,0,0.6450133323669434
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,40,128,1,float16,float16,0,10.437786738077799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,2,128,1,float16,float16,0,8.780192057291666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,1,128,1,float16,float16,0,8.67087491353353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,1,128,1,float16,float16,0,4.177381197611491
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,40,4,128,1,float16,float16,0,18.081019083658855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,40,1,128,1,float16,float16,0,17.871978759765625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,40,128,1,float16,float16,0,4.981034596761067
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,4,128,1,float16,float16,0,9.402448018391928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,8,128,1,float16,float16,0,9.525210698445639
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,40,128,1,float16,float16,0,2.4766507148742676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,2,128,1,float16,float16,0,4.2344160079956055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,4,128,1,float16,float16,0,4.443354606628418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,40,2,128,1,float16,float16,0,17.905045827229817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,2,128,1,float16,float16,0,2.182362715403239
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,4,128,1,float16,float16,0,2.224720001220703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,40,8,128,1,float16,float16,0,19.10983403523763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,8,128,1,float16,float16,0,4.46833070119222
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,40,128,1,float16,float16,0,1.2779893080393474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,4,128,1,float16,float16,0,1.1374879678090413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,2,128,1,float16,float16,0,1.0898880163828533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,1,128,1,float16,float16,0,2.067509333292643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,40,128,1,float16,float16,0,0.6629120111465454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,2,128,1,float16,float16,0,0.5623466571172079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,8,128,1,float16,float16,0,2.228389263153076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,1,128,1,float16,float16,0,0.5555413166681925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,40,128,1,float16,float16,0,0.3521653413772583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,1,128,1,float16,float16,0,0.29847999413808185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,4,128,1,float16,float16,0,0.5902986526489258
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,8,128,1,float16,float16,0,0.5989600022633871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,4,128,1,float16,float16,0,0.3197279969851176
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,8,128,1,float16,float16,0,0.3203519980112712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,2,128,1,float16,float16,0,0.3067359924316406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,8,128,1,float16,float16,0,1.147157351175944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,1,128,1,float16,float16,0,1.0534559885660808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,1,128,1,float16,float16,0,4.964170773824056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,4,128,1,float16,float16,0,5.465808232625325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,40,128,1,float16,float16,0,6.236256281534831
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,2,128,1,float16,float16,0,5.156197230021159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,40,1,128,1,float16,float16,0,10.607264200846354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,40,128,1,float16,float16,0,3.0714826583862305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,40,4,128,1,float16,float16,0,11.410672505696615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,1,128,1,float16,float16,0,2.4752000172932944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,40,8,128,1,float16,float16,0,11.377423604329428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,2,128,1,float16,float16,0,2.6451093355814614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,8,128,1,float16,float16,0,5.5631148020426435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,1,128,1,float16,float16,0,1.226362705230713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,2,128,1,float16,float16,0,1.3237653573354085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,40,128,1,float16,float16,0,1.5959787368774414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,8,128,1,float16,float16,0,2.791365305582682
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,4,128,1,float16,float16,0,1.3736480077107747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,4,128,1,float16,float16,0,2.7322187423706055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,8,128,1,float16,float16,0,1.459664026896159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,40,2,128,1,float16,float16,0,10.884234110514322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,40,128,1,float16,float16,0,0.7933226426442465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,1,128,1,float16,float16,0,0.6354613304138184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,4,128,1,float16,float16,0,0.6930987040201823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,40,128,1,float16,float16,0,0.41364800930023193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,8,128,1,float16,float16,0,0.6983199914296468
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,8,128,1,float16,float16,0,0.37054399649302167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,1,128,1,float16,float16,0,0.3375466664632161
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,40,128,1,float16,float16,0,0.22660799821217856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,4,128,1,float16,float16,0,0.3636426528294881
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,1,128,1,float16,float16,0,0.18722132841746011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,2,128,1,float16,float16,0,0.656005342801412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,2,128,1,float16,float16,0,0.19236266613006592
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,4,128,1,float16,float16,0,0.20055466890335083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,8,128,1,float16,float16,0,0.20478399594624838
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,2,128,1,float16,float16,0,0.34889066219329834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,1,128,1,float16,float16,0,4.922325452168782
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,40,128,1,float16,float16,0,6.4286454518636065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,2,128,1,float16,float16,0,4.871813456217448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,40,1,128,1,float16,float16,0,10.125018437703451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,8,128,1,float16,float16,0,5.519258499145508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,4,128,1,float16,float16,0,5.6389815012613935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,1,128,1,float16,float16,0,2.4203893343607583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,40,128,1,float16,float16,0,3.224757194519043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,2,128,1,float16,float16,0,2.5256266593933105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,40,128,1,float16,float16,0,1.6160160700480144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,4,128,1,float16,float16,0,2.760554631551107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,1,128,1,float16,float16,0,1.2000373204549153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,8,128,1,float16,float16,0,2.827594757080078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,2,128,1,float16,float16,0,1.2662399609883626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,40,2,128,1,float16,float16,0,10.679115295410156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,40,4,128,1,float16,float16,0,11.490352630615234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,4,128,1,float16,float16,0,1.353376070658366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,8,128,1,float16,float16,0,1.377061367034912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,40,128,1,float16,float16,0,0.8199520111083984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,1,128,1,float16,float16,0,0.6132959922154745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,2,128,1,float16,float16,0,0.66976531346639
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,40,128,1,float16,float16,0,0.4264479875564575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,2,128,1,float16,float16,0,0.3349333206812541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,1,128,1,float16,float16,0,0.3245226740837097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,40,8,128,1,float16,float16,0,11.553221384684244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,8,128,1,float16,float16,0,0.36698134740193683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,8,128,1,float16,float16,0,0.7013386885325114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,4,128,1,float16,float16,0,0.3681600093841553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,40,128,1,float16,float16,0,0.22926932573318481
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,1,128,1,float16,float16,0,0.17744000752766928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,8,128,1,float16,float16,0,0.1972000002861023
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,1,128,1,float16,float16,0,0.09988266229629517
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,4,128,1,float16,float16,0,0.19591999053955078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,40,128,1,float16,float16,0,0.1306773324807485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,2,128,1,float16,float16,0,0.10363733768463135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,8,128,1,float16,float16,0,0.10925333698590596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,4,128,1,float16,float16,0,0.6990666389465332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,4,128,1,float16,float16,0,0.10905599594116211
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,2,128,1,float16,float16,0,0.18623999754587808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,1,128,1,float16,float16,0,3.0654026667277017
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,40,128,1,float16,float16,0,4.682538668314616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,2,128,1,float16,float16,0,3.2630399068196616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,40,1,128,1,float16,float16,0,6.5091203053792315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,40,2,128,1,float16,float16,0,6.735866546630859
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,40,8,128,1,float16,float16,0,7.5286610921223955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,40,128,1,float16,float16,0,2.323194662729899
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,1,128,1,float16,float16,0,1.5400586128234863
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,8,128,1,float16,float16,0,3.713871955871582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,2,128,1,float16,float16,0,1.67685333887736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,4,128,1,float16,float16,0,3.6422719955444336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,40,128,1,float16,float16,0,1.177242676417033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,1,128,1,float16,float16,0,0.771882692972819
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,4,128,1,float16,float16,0,0.9336053530375162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,1,128,1,float16,float16,0,0.3967680136362712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,40,128,1,float16,float16,0,0.5976959864298502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,4,128,1,float16,float16,0,0.472053329149882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,40,4,128,1,float16,float16,0,7.5762983957926435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,8,128,1,float16,float16,0,0.9297813574473063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,2,128,1,float16,float16,0,0.4241226514180501
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,8,128,1,float16,float16,0,0.47621333599090576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,4,128,1,float16,float16,0,1.8274240493774414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,40,128,1,float16,float16,0,0.31098665793736774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,4,128,1,float16,float16,0,0.24537599086761475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,1,128,1,float16,float16,0,0.1165013313293457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,8,128,1,float16,float16,0,0.2483839988708496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,2,128,1,float16,float16,0,0.12050132950146993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,4,128,1,float16,float16,0,0.13386666774749756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,8,128,1,float16,float16,0,0.1341600020726522
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,40,128,1,float16,float16,0,0.09412266810735066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,1,128,1,float16,float16,0,0.20645334323247275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,1,128,1,float16,float16,0,0.06586133440335591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,2,128,1,float16,float16,0,0.8210986455281576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,40,128,1,float16,float16,0,0.05657599866390228
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,8,128,1,float16,float16,0,0.07554133236408234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,4,128,1,float16,float16,0,0.07320533196131389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,2,128,1,float16,float16,0,0.0681279997030894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,1,128,1,float16,float16,0,0.04277333120505015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,40,128,1,float16,float16,0,0.16762133439381918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,2,128,1,float16,float16,0,0.04433066646258036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,8,128,1,float16,float16,0,0.04562133550643921
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,4,128,1,float16,float16,0,0.045509333411852516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,8,128,1,float16,float16,0,1.8496373494466145
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,2,128,1,float16,float16,0,0.22192533810933432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,1,128,1,float16,float16,0,1.0858240127563477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,40,128,1,float16,float16,0,1.9074613253275554
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,2,128,1,float16,float16,0,1.1858346462249756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,40,128,1,float16,float16,0,0.959824005762736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,40,2,128,1,float16,float16,0,2.3826187451680503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,1,128,1,float16,float16,0,0.5342773199081421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,40,1,128,1,float16,float16,0,2.2159999211629233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,4,128,1,float16,float16,0,1.398431936899821
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,8,128,1,float16,float16,0,1.417525291442871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,8,128,1,float16,float16,0,0.7062133153279623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,2,128,1,float16,float16,0,0.5874346494674683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,4,128,1,float16,float16,0,0.6941493352254232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,40,128,1,float16,float16,0,0.5007893244425455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,40,4,128,1,float16,float16,0,2.809413274129232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,40,8,128,1,float16,float16,0,2.856368064880371
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,1,128,1,float16,float16,0,0.2754986683527629
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,2,128,1,float16,float16,0,0.30144532521565753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,4,128,1,float16,float16,0,0.35470934708913165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,40,128,1,float16,float16,0,0.2529013355573018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,1,128,1,float16,float16,0,0.14884266257286072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,8,128,1,float16,float16,0,0.36309866110483807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,40,128,1,float16,float16,0,0.137061337629954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,1,128,1,float16,float16,0,0.0824533353249232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,4,128,1,float16,float16,0,0.18437333901723227
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,2,128,1,float16,float16,0,0.08745066324869792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,4,128,1,float16,float16,0,0.10186133782068889
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,40,128,1,float16,float16,0,0.07659733295440674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,1,128,1,float16,float16,0,0.045381332437197365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,4,128,1,float16,float16,0,0.053210665782292686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,8,128,1,float16,float16,0,0.10243200262387593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,2,128,1,float16,float16,0,0.04889066517353058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,40,128,1,float16,float16,0,0.043231998880704246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,1,128,1,float16,float16,0,0.029813334345817566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,8,128,1,float16,float16,0,0.05811200042565664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,2,128,1,float16,float16,0,0.03044266750415166
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,4,128,1,float16,float16,0,0.03253866732120514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,2,128,1,float16,float16,0,0.1600266695022583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,40,128,1,float16,float16,0,0.025850666066010792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,2,128,1,float16,float16,0,0.021520001192887623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,8,128,1,float16,float16,0,0.03272533416748047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,1,128,1,float16,float16,0,0.021957332889238994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,4,128,1,float16,float16,0,0.023082666099071503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,8,128,1,float16,float16,0,0.023157333334287006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,8,128,1,float16,float16,0,0.18942399819691977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,1,128,1,float16,float16,0,0.4127306540807088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,40,128,1,float16,float16,0,0.9061173597971598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,2,128,1,float16,float16,0,0.4729013442993164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,40,1,128,1,float16,float16,0,0.9607199827829996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,40,2,128,1,float16,float16,0,1.0687893231709797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,40,128,1,float16,float16,0,0.4539253314336141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,4,128,1,float16,float16,0,0.579749345779419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,1,128,1,float16,float16,0,0.21315733591715494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,40,4,128,1,float16,float16,0,1.302448034286499
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,2,128,1,float16,float16,0,0.2413653333981832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,8,128,1,float16,float16,0,0.5953866640726725
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,8,128,1,float16,float16,0,0.3036053379376729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,4,128,1,float16,float16,0,0.2945493261019389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,40,128,1,float16,float16,0,0.23833600680033365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,1,128,1,float16,float16,0,0.11406399806340535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,2,128,1,float16,float16,0,0.12733866771062216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,40,128,1,float16,float16,0,0.12625599900881448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,1,128,1,float16,float16,0,0.06550399959087372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,4,128,1,float16,float16,0,0.08406399687131245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,2,128,1,float16,float16,0,0.07320533196131389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,8,128,1,float16,float16,0,0.1569386621316274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,8,128,1,float16,float16,0,0.08604799707730611
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,4,128,1,float16,float16,0,0.15308266878128052
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,40,128,1,float16,float16,0,0.07021333277225494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,1,128,1,float16,float16,0,0.03493333359559377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,2,128,1,float16,float16,0,0.036757332583268486
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,40,8,128,1,float16,float16,0,1.3307946523030598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,4,128,1,float16,float16,0,0.041722665230433144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,8,128,1,float16,float16,0,0.04500266909599304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,1,128,1,float16,float16,0,0.023061332603295643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,2,128,1,float16,float16,0,0.02404266595840454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,4,128,1,float16,float16,0,0.025461333493391674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,40,128,1,float16,float16,0,0.039279999832312264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,40,128,1,float16,float16,0,0.02038399999340375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,2,128,1,float16,float16,0,0.016613333175579708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,1,128,1,float16,float16,0,0.016607999801635742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,4,128,1,float16,float16,0,0.018298666924238205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,1,128,1,float16,float16,0,0.01310933381319046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,8,128,1,float16,float16,0,0.01794133335351944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,2,128,1,float16,float16,0,0.012826666235923767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,4,128,1,float16,float16,0,0.013797332843144735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,8,128,1,float16,float16,0,0.013327999661366144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,40,128,1,float16,float16,0,0.014815999815861383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,8,128,1,float16,float16,0,0.02604266752799352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,2,128,1,float16,float16,0,0.23748799165089926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,1,128,1,float16,float16,0,0.20977065960566202
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,4,128,1,float16,float16,0,0.29155733187993366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,40,1,128,1,float16,float16,0,0.4020266532897949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,40,128,1,float16,float16,0,0.45482667287190753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,40,4,128,1,float16,float16,0,0.5662613312403361
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,40,2,128,1,float16,float16,0,0.45901334285736084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,8,128,1,float16,float16,0,0.29977599779764813
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,40,8,128,1,float16,float16,0,0.5862133502960205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,1,128,1,float16,float16,0,0.11514133214950562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,40,128,1,float16,float16,0,0.23859200874964395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,4,128,1,float16,float16,0,0.0846506655216217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,2,128,1,float16,float16,0,0.12760532895723978
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,40,128,1,float16,float16,0,0.07036266724268596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,1,128,1,float16,float16,0,0.06633066634337108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,8,128,1,float16,float16,0,0.08629332979520161
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,2,128,1,float16,float16,0,0.0360000009338061
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,40,128,1,float16,float16,0,0.12487467130025227
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,1,128,1,float16,float16,0,0.0328053335348765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,4,128,1,float16,float16,0,0.04031999905904134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,8,128,1,float16,float16,0,0.15582399566968283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,4,128,1,float16,float16,0,0.15241066614786783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,40,128,1,float16,float16,0,0.03873066604137421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,2,128,1,float16,float16,0,0.022469334304332733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,4,128,1,float16,float16,0,0.02476266771554947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,8,128,1,float16,float16,0,0.04586666822433472
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,4,128,1,float16,float16,0,0.017738666385412216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,2,128,1,float16,float16,0,0.015941333025693893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,8,128,1,float16,float16,0,0.025050667424996693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,8,128,1,float16,float16,0,0.01782400036851565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,1,128,1,float16,float16,0,0.01598400001724561
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,2,128,1,float16,float16,0,0.07108266651630402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,1,128,1,float16,float16,0,0.021210665504137676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,40,128,1,float16,float16,0,0.020367999871571858
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,40,128,1,float16,float16,0,0.01470400020480156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,1,128,1,float16,float16,0,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,2,128,1,float16,float16,0,0.012453333785136541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,4,128,1,float16,float16,0,0.013183999806642532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,2,128,1,float16,float16,0,0.012069333344697952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,8,128,1,float16,float16,0,0.012960000584522883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,4,128,1,float16,float16,0,0.012250666817029318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,8,128,1,float16,float16,0,0.012250666817029318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,40,128,1,float16,float16,0,0.013157332936922709
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,1,128,1,float16,float16,0,0.194106658299764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,2,128,1,float16,float16,0,0.20690133174260458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,40,1,128,1,float16,float16,0,0.3755520184834798
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,40,4,128,1,float16,float16,0,0.4484853347142537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,4,128,1,float16,float16,0,0.22963732481002808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,8,128,1,float16,float16,0,0.23301867643992105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,40,2,128,1,float16,float16,0,0.398911992708842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,40,128,1,float16,float16,0,0.29554132620493573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,40,128,1,float16,float16,0,0.1539413332939148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,1,128,1,float16,float16,0,0.10365866621335347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,2,128,1,float16,float16,0,0.10833066701889038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,4,128,1,float16,float16,0,0.12140267093976338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,40,8,128,1,float16,float16,0,0.4520639975865682
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,1,128,1,float16,float16,0,0.05123200019200643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,4,128,1,float16,float16,0,0.058320000767707825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,2,128,1,float16,float16,0,0.053557331363360085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,8,128,1,float16,float16,0,0.06282133360703786
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,8,128,1,float16,float16,0,0.12246933579444885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,40,128,1,float16,float16,0,0.08479467034339905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,1,128,1,float16,float16,0,0.02871999889612198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,2,128,1,float16,float16,0,0.03019733230272929
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,4,128,1,float16,float16,0,0.03266133368015289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,40,128,1,float16,float16,0,0.04494399825731913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,8,128,1,float16,float16,0,0.032618666688601174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,2,128,1,float16,float16,0,0.0194560003777345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,40,128,1,float16,float16,0,0.024533333877722423
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,4,128,1,float16,float16,0,0.021333334346612293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,8,128,1,float16,float16,0,0.021909333765506744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,1,128,1,float16,float16,0,0.014842666685581207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,2,128,1,float16,float16,0,0.014560000350077948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,4,128,1,float16,float16,0,0.014981333166360855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,40,128,1,float16,float16,0,0.016704000532627106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,1,128,1,float16,float16,0,0.01942933350801468
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,8,128,1,float16,float16,0,0.015082667271296183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,40,128,1,float16,float16,0,0.01321600005030632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,2,128,1,float16,float16,0,0.01210133358836174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,1,128,1,float16,float16,0,0.012069333344697952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,4,128,1,float16,float16,0,0.011957333733638128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,8,128,1,float16,float16,0,0.012096000214417776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,40,128,1,float16,float16,0,0.012682666381200155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,1,128,1,float16,float16,0,0.01180800050497055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,2,128,1,float16,float16,0,0.011584000041087469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,4,128,1,float16,float16,0,0.011909333368142446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,8,128,1,float16,float16,0,0.011909333368142446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,1,128,1,float16,float16,0,0.18425067265828451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,2,128,1,float16,float16,0,0.19085333744684854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,4,128,1,float16,float16,0,0.20376533269882202
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,40,128,1,float16,float16,0,0.12304000059763591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,1,128,1,float16,float16,0,0.09037333726882935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,40,2,128,1,float16,float16,0,0.3725546598434448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,40,128,1,float16,float16,0,0.23625600337982178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,40,4,128,1,float16,float16,0,0.3964373270670573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,8,128,1,float16,float16,0,0.20426666736602783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,40,1,128,1,float16,float16,0,0.3622613350550334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,40,8,128,1,float16,float16,0,0.40067732334136963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,2,128,1,float16,float16,0,0.09251733620961507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,4,128,1,float16,float16,0,0.09819733103116353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,2,128,1,float16,float16,0,0.04969066878159841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,1,128,1,float16,float16,0,0.04824000100294749
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,40,128,1,float16,float16,0,0.06599466502666473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,40,128,1,float16,float16,0,0.03329066683848699
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,8,128,1,float16,float16,0,0.10339732964833577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,1,128,1,float16,float16,0,0.02829333394765854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,4,128,1,float16,float16,0,0.05183466772238413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,2,128,1,float16,float16,0,0.02827200045188268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,4,128,1,float16,float16,0,0.030213333666324615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,8,128,1,float16,float16,0,0.05204799771308899
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,40,128,1,float16,float16,0,0.021290667355060577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,1,128,1,float16,float16,0,0.019082666685183842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,8,128,1,float16,float16,0,0.030426666140556335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,2,128,1,float16,float16,0,0.018911999960740406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,4,128,1,float16,float16,0,0.0195573332409064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,8,128,1,float16,float16,0,0.019434666881958645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,40,128,1,float16,float16,0,0.015450666348139444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,2,128,1,float16,float16,0,0.014335999886194864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,1,128,1,float16,float16,0,0.014245333770910898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,4,128,1,float16,float16,0,0.014416000495354334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,2,128,1,float16,float16,0,0.01180800050497055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,40,128,1,float16,float16,0,0.012917333592971167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,8,128,1,float16,float16,0,0.012042666474978128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,8,128,1,float16,float16,0,0.014538666854302088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,1,128,1,float16,float16,0,0.011722666521867117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,4,128,1,float16,float16,0,0.01184533288081487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,40,128,1,float16,float16,0,0.012341332932313284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,1,128,1,float16,float16,0,0.011519999553759893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,2,128,1,float16,float16,0,0.011440000186363855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,4,128,1,float16,float16,0,0.011781333635250727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,8,128,1,float16,float16,0,0.011701333026091257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,40,128,1,float16,float16,0,0.1616159975528717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,0,0.016554666062196095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,0,0.03791466603676478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,0,0.03057066599527995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,0,0.09062400460243225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,0,0.021920000513394673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,0,0.059631998340288796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,0,0.042352000872294106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,0,0.053039997816085815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,0,0.013850666582584381
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,0,0.023381332556406658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,40,128,1,float16,float16,0,0.08518399794896443
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,0,0.013264000415802002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,40,128,1,float16,float16,0,0.04738666613896688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,0,0.020469332734743755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,0,0.022869333624839783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,0,0.027727998793125153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,0,0.03422933320204417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,0,0.02701866626739502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,40,128,1,float16,float16,0,0.02812800059715907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,0,0.013056000073750814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,0,0.019866666446129482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,0,0.020053333292404812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,0,0.022197333474953968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,0,0.01941866676012675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,0,0.019285333653291065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,40,128,1,float16,float16,0,0.01878400022784869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,0,0.012565333396196365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,0,0.018618666877349217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,40,128,1,float16,float16,0,0.014581333845853806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,0,0.018874666343132656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,0,0.019482667247454327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,0,0.018911999960740406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,40,128,1,float16,float16,0,0.011968000481526056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,0,0.01258133351802826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,0,0.018239999810854595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,0,0.01863466699918111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,0,0.012389333297808966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,0,0.050293331344922386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,0,0.01250133290886879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,0,0.01246400053302447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,40,128,1,float16,float16,0,0.011871999750534693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,0,0.012560000022252401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,1,128,1,float16,float16,0,49.00365193684896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,2,128,1,float16,float16,0,49.29704284667969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,32,128,1,float16,float16,0,50.890096028645836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,4,128,1,float16,float16,0,49.28333028157552
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,32,128,1,float16,float16,0,25.26812235514323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,1,128,1,float16,float16,0,25.544784545898438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,2,128,1,float16,float16,0,24.511072794596355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,32,2,128,1,float16,float16,0,97.94810994466145
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,32,1,128,1,float16,float16,0,98.02858479817708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,32,4,128,1,float16,float16,0,99.48802693684895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,32,8,128,1,float16,float16,0,100.16379801432292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,8,128,1,float16,float16,0,50.6239267985026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,32,128,1,float16,float16,0,12.554395039876303
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,4,128,1,float16,float16,0,25.000864664713543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,8,128,1,float16,float16,0,25.37353515625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,1,128,1,float16,float16,0,12.207962036132812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,2,128,1,float16,float16,0,12.41094970703125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,4,128,1,float16,float16,0,12.478293100992838
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,8,128,1,float16,float16,0,12.505157470703125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,1,128,1,float16,float16,0,27.84251658121745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,32,128,1,float16,float16,0,28.482927958170574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,2,128,1,float16,float16,0,28.051294962565105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,4,128,1,float16,float16,0,28.357269287109375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,32,128,1,float16,float16,0,14.291941324869791
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,1,128,1,float16,float16,0,14.015546162923178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,2,128,1,float16,float16,0,13.953973134358725
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,32,1,128,1,float16,float16,0,56.03269958496094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,32,2,128,1,float16,float16,0,55.224995930989586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,32,4,128,1,float16,float16,0,57.06756591796875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,32,8,128,1,float16,float16,0,57.54075622558594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,32,128,1,float16,float16,0,7.216874440511067
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,1,128,1,float16,float16,0,6.97541872660319
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,8,128,1,float16,float16,0,28.439926147460938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,2,128,1,float16,float16,0,6.7790877024332685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,4,128,1,float16,float16,0,7.089424133300781
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,4,128,1,float16,float16,0,14.111887613932291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,8,128,1,float16,float16,0,7.186682383219401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,8,128,1,float16,float16,0,14.23300298055013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,1,128,1,float16,float16,0,19.358394622802734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,32,128,1,float16,float16,0,20.291610717773438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,2,128,1,float16,float16,0,19.99193064371745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,4,128,1,float16,float16,0,19.694469451904297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,32,128,1,float16,float16,0,10.198469161987305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,1,128,1,float16,float16,0,9.823354721069336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,32,1,128,1,float16,float16,0,38.783851623535156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,2,128,1,float16,float16,0,9.847978591918945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,32,2,128,1,float16,float16,0,39.08329518636068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,32,4,128,1,float16,float16,0,39.72876230875651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,32,128,1,float16,float16,0,5.08020814259847
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,8,128,1,float16,float16,0,19.696762084960938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,1,128,1,float16,float16,0,4.834831873575847
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,2,128,1,float16,float16,0,4.921669324239095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,32,8,128,1,float16,float16,0,39.32088979085287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,4,128,1,float16,float16,0,9.976645151774088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,8,128,1,float16,float16,0,9.995264053344727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,4,128,1,float16,float16,0,4.936309178670247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,8,128,1,float16,float16,0,4.897765477498372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,1,128,1,float16,float16,0,25.39911397298177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,32,128,1,float16,float16,0,26.38255564371745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,2,128,1,float16,float16,0,25.316009521484375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,4,128,1,float16,float16,0,25.94800567626953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,32,128,1,float16,float16,0,13.227466583251953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,1,128,1,float16,float16,0,12.580212910970053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,2,128,1,float16,float16,0,12.70749282836914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,32,1,128,1,float16,float16,0,50.0118153889974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,32,2,128,1,float16,float16,0,50.679972330729164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,32,128,1,float16,float16,0,6.508277257283528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,32,4,128,1,float16,float16,0,51.7851308186849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,32,8,128,1,float16,float16,0,51.22804260253906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,8,128,1,float16,float16,0,26.12615458170573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,8,128,1,float16,float16,0,12.942933400472006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,1,128,1,float16,float16,0,6.206842422485352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,4,128,1,float16,float16,0,12.94488525390625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,32,128,1,float16,float16,0,3.3665119806925454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,2,128,1,float16,float16,0,6.276767730712891
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,4,128,1,float16,float16,0,6.464714686075847
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,8,128,1,float16,float16,0,6.267194747924805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,1,128,1,float16,float16,0,3.1383094787597656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,2,128,1,float16,float16,0,3.1277812321980796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,4,128,1,float16,float16,0,3.25981871287028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,8,128,1,float16,float16,0,3.1200106938680015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,1,128,1,float16,float16,0,14.406421661376953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,32,128,1,float16,float16,0,15.206981658935547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,2,128,1,float16,float16,0,14.698384602864584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,32,128,1,float16,float16,0,7.503706614176433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,1,128,1,float16,float16,0,7.077168146769206
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,32,1,128,1,float16,float16,0,28.960784912109375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,32,2,128,1,float16,float16,0,29.323265075683594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,32,4,128,1,float16,float16,0,29.349258422851562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,32,8,128,1,float16,float16,0,29.70258076985677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,4,128,1,float16,float16,0,14.87545649210612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,2,128,1,float16,float16,0,7.161594390869141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,32,128,1,float16,float16,0,3.806298573811849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,8,128,1,float16,float16,0,14.697867075602213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,4,128,1,float16,float16,0,7.432079950968425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,1,128,1,float16,float16,0,3.459749221801758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,2,128,1,float16,float16,0,3.5606559117635093
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,4,128,1,float16,float16,0,3.701903978983561
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,8,128,1,float16,float16,0,7.353237152099609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,32,128,1,float16,float16,0,1.9654879570007324
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,2,128,1,float16,float16,0,1.8885280291239421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,8,128,1,float16,float16,0,3.634592056274414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,1,128,1,float16,float16,0,1.8087093035380046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,4,128,1,float16,float16,0,1.8505013783772786
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,8,128,1,float16,float16,0,1.8844960530598958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,2,128,1,float16,float16,0,13.29202651977539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,4,128,1,float16,float16,0,13.808245340983072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,32,128,1,float16,float16,0,14.08447519938151
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,1,128,1,float16,float16,0,13.19717280069987
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,1,128,1,float16,float16,0,6.4314829508463545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,32,128,1,float16,float16,0,7.092842737833659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,2,128,1,float16,float16,0,6.469807942708333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,32,1,128,1,float16,float16,0,26.480016072591145
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,32,2,128,1,float16,float16,0,26.90338134765625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,32,4,128,1,float16,float16,0,27.70110829671224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,32,128,1,float16,float16,0,3.4761174519856772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,32,8,128,1,float16,float16,0,27.882596333821613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,8,128,1,float16,float16,0,13.713733673095703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,4,128,1,float16,float16,0,6.6868642171223955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,8,128,1,float16,float16,0,6.81379763285319
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,1,128,1,float16,float16,0,3.139888127644857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,2,128,1,float16,float16,0,3.3245652516682944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,32,128,1,float16,float16,0,1.8139360745747883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,1,128,1,float16,float16,0,1.6281545956929524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,4,128,1,float16,float16,0,3.363722801208496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,2,128,1,float16,float16,0,1.6508533159891765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,8,128,1,float16,float16,0,3.3958508173624673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,32,128,1,float16,float16,0,0.9362186590830485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,4,128,1,float16,float16,0,1.7355093955993652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,1,128,1,float16,float16,0,0.8624373277028402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,2,128,1,float16,float16,0,0.8585226535797119
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,8,128,1,float16,float16,0,1.7373332977294922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,4,128,1,float16,float16,0,0.9065866470336914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,8,128,1,float16,float16,0,0.899616003036499
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,1,128,1,float16,float16,0,7.624741236368815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,2,128,1,float16,float16,0,7.831525166829427
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,4,128,1,float16,float16,0,7.942901611328125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,32,128,1,float16,float16,0,7.930351893107097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,1,128,1,float16,float16,0,3.669343948364258
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,32,128,1,float16,float16,0,4.144805272420247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,32,1,128,1,float16,float16,0,15.19485855102539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,32,2,128,1,float16,float16,0,15.5489870707194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,2,128,1,float16,float16,0,3.698495864868164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,32,4,128,1,float16,float16,0,16.159183502197266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,32,8,128,1,float16,float16,0,16.37993621826172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,8,128,1,float16,float16,0,8.027626673380533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,1,128,1,float16,float16,0,1.8731412887573242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,4,128,1,float16,float16,0,3.9110240936279297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,8,128,1,float16,float16,0,3.9251041412353516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,32,128,1,float16,float16,0,2.066330591837565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,2,128,1,float16,float16,0,1.8625013033548992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,4,128,1,float16,float16,0,1.99727996190389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,32,128,1,float16,float16,0,1.1128693421681721
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,1,128,1,float16,float16,0,0.9709813594818115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,2,128,1,float16,float16,0,0.9904586474100748
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,8,128,1,float16,float16,0,2.0205440521240234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,4,128,1,float16,float16,0,1.0201226870218914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,8,128,1,float16,float16,0,1.0054773489634197
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,1,128,1,float16,float16,0,0.5151840051015218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,2,128,1,float16,float16,0,0.5250080029169718
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,4,128,1,float16,float16,0,0.5400746663411459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,32,128,1,float16,float16,0,0.5585439999898275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,8,128,1,float16,float16,0,0.5439573526382446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,1,128,1,float16,float16,0,7.153807957967122
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,4,128,1,float16,float16,0,7.586245218912761
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,2,128,1,float16,float16,0,7.119093577067058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,32,128,1,float16,float16,0,4.002655982971191
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,32,128,1,float16,float16,0,8.258997599283854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,32,1,128,1,float16,float16,0,14.430447896321615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,32,4,128,1,float16,float16,0,15.432351430257162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,32,2,128,1,float16,float16,0,14.89178721110026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,32,8,128,1,float16,float16,0,15.523712158203125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,8,128,1,float16,float16,0,7.6530507405598955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,1,128,1,float16,float16,0,3.412032127380371
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,1,128,1,float16,float16,0,1.720853328704834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,2,128,1,float16,float16,0,1.7577600479125977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,8,128,1,float16,float16,0,3.696117401123047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,4,128,1,float16,float16,0,3.674799919128418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,4,128,1,float16,float16,0,1.850554625193278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,1,128,1,float16,float16,0,0.8762666384379069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,2,128,1,float16,float16,0,0.9029173056284586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,32,128,1,float16,float16,0,1.0155573685963948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,8,128,1,float16,float16,0,1.8356374104817708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,2,128,1,float16,float16,0,3.41979185740153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,32,128,1,float16,float16,0,1.9644160270690918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,4,128,1,float16,float16,0,0.934122641881307
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,8,128,1,float16,float16,0,0.4974026679992676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,1,128,1,float16,float16,0,0.453109343846639
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,8,128,1,float16,float16,0,0.9445919990539551
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,2,128,1,float16,float16,0,0.4679679870605469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,32,128,1,float16,float16,0,0.528874675432841
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,32,128,1,float16,float16,0,0.28698132435480755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,2,128,1,float16,float16,0,0.2581760088602702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,1,128,1,float16,float16,0,0.24835733572642008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,4,128,1,float16,float16,0,0.27078932523727417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,8,128,1,float16,float16,0,0.27292267481486004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,4,128,1,float16,float16,0,0.4956586758295695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,1,128,1,float16,float16,0,3.9733654657999673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,32,128,1,float16,float16,0,4.930885314941406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,4,128,1,float16,float16,0,4.4508107503255205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,32,128,1,float16,float16,0,2.462560017903646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,1,128,1,float16,float16,0,2.024160067240397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,32,4,128,1,float16,float16,0,9.319477081298828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,32,1,128,1,float16,float16,0,8.45463498433431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,32,2,128,1,float16,float16,0,8.592421213785807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,32,8,128,1,float16,float16,0,9.361562728881836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,8,128,1,float16,float16,0,4.540314674377441
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,2,128,1,float16,float16,0,4.125199953715007
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,4,128,1,float16,float16,0,1.1534773508707683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,2,128,1,float16,float16,0,1.0607146422068279
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,2,128,1,float16,float16,0,2.086479981740316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,8,128,1,float16,float16,0,2.278421401977539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,4,128,1,float16,float16,0,2.199397404988607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,32,128,1,float16,float16,0,1.255130688349406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,1,128,1,float16,float16,0,1.0113120079040527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,32,128,1,float16,float16,0,0.6349120140075684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,4,128,1,float16,float16,0,0.5869066715240479
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,1,128,1,float16,float16,0,0.5193759997685751
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,32,128,1,float16,float16,0,0.34331734975179035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,8,128,1,float16,float16,0,1.1540746688842773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,2,128,1,float16,float16,0,0.5416053136189779
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,1,128,1,float16,float16,0,0.28380799293518066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,8,128,1,float16,float16,0,0.5914880037307739
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,2,128,1,float16,float16,0,0.2934826612472534
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,4,128,1,float16,float16,0,0.30940266450246173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,8,128,1,float16,float16,0,0.3112480044364929
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,32,128,1,float16,float16,0,0.18927466869354248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,2,128,1,float16,float16,0,0.16262933611869812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,4,128,1,float16,float16,0,0.169813334941864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,8,128,1,float16,float16,0,0.1752906640370687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,1,128,1,float16,float16,0,0.15995200475056967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,32,128,1,float16,float16,0,5.143424034118652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,1,128,1,float16,float16,0,3.8707520167032876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,2,128,1,float16,float16,0,4.026010513305664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,32,128,1,float16,float16,0,2.519754727681478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,32,8,128,1,float16,float16,0,9.563242594401041
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,1,128,1,float16,float16,0,1.9603039423624675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,8,128,1,float16,float16,0,4.528784116109212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,32,1,128,1,float16,float16,0,7.7860158284505205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,32,2,128,1,float16,float16,0,8.398037592569986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,2,128,1,float16,float16,0,2.021125316619873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,8,128,1,float16,float16,0,2.2675253550211587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,4,128,1,float16,float16,0,4.467610677083333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,1,128,1,float16,float16,0,1.0403786500295003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,32,128,1,float16,float16,0,1.2972853183746338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,4,128,1,float16,float16,0,1.1165013313293457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,2,128,1,float16,float16,0,1.0604320367177327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,1,128,1,float16,float16,0,0.5170026620229086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,8,128,1,float16,float16,0,1.1431466738382976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,4,128,1,float16,float16,0,2.2747839291890464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,2,128,1,float16,float16,0,0.5267999966939291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,32,128,1,float16,float16,0,0.3458933432896932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,4,128,1,float16,float16,0,0.5745333433151245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,1,128,1,float16,float16,0,0.2685226599375407
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,8,128,1,float16,float16,0,0.5927786827087402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,4,128,1,float16,float16,0,0.3048853278160095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,8,128,1,float16,float16,0,0.3083413243293762
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,2,128,1,float16,float16,0,0.2878986597061157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,32,128,1,float16,float16,0,0.6558293501536051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,32,128,1,float16,float16,0,0.18282665808995566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,2,128,1,float16,float16,0,0.1555359959602356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,1,128,1,float16,float16,0,0.14965867002805075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,8,128,1,float16,float16,0,0.16743467251459757
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,4,128,1,float16,float16,0,0.16890132427215576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,4,128,1,float16,float16,0,0.09142399827639262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,2,128,1,float16,float16,0,0.086325337489446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,32,4,128,1,float16,float16,0,9.325818379720053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,32,128,1,float16,float16,0,0.10997866590817769
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,8,128,1,float16,float16,0,0.09672000010808308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,1,128,1,float16,float16,0,0.08660800258318584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,32,128,1,float16,float16,0,3.696432113647461
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,1,128,1,float16,float16,0,2.463717301686605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,2,128,1,float16,float16,0,2.656005382537842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,32,1,128,1,float16,float16,0,5.121909459431966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,1,128,1,float16,float16,0,1.2827200094858806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,4,128,1,float16,float16,0,3.1097707748413086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,8,128,1,float16,float16,0,3.133061408996582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,32,128,1,float16,float16,0,1.858682632446289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,32,2,128,1,float16,float16,0,5.370080312093099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,32,4,128,1,float16,float16,0,6.217386881510417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,2,128,1,float16,float16,0,1.3618133862813313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,32,8,128,1,float16,float16,0,6.336629231770833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,32,128,1,float16,float16,0,0.9349439938863119
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,4,128,1,float16,float16,0,0.7868586381276449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,4,128,1,float16,float16,0,1.5611999829610188
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,32,128,1,float16,float16,0,0.47465598583221436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,1,128,1,float16,float16,0,0.32227200269699097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,2,128,1,float16,float16,0,0.35438934961954754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,8,128,1,float16,float16,0,1.5852479934692383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,8,128,1,float16,float16,0,0.40655465920766193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,4,128,1,float16,float16,0,0.40480534235636395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,1,128,1,float16,float16,0,0.17267733812332153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,32,128,1,float16,float16,0,0.24929600954055786
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,1,128,1,float16,float16,0,0.6319146553675333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,32,128,1,float16,float16,0,0.13482667009035745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,8,128,1,float16,float16,0,0.215013325214386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,2,128,1,float16,float16,0,0.1042133371035258
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,1,128,1,float16,float16,0,0.0981226662794749
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,4,128,1,float16,float16,0,0.11542399724324544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,4,128,1,float16,float16,0,0.21029333273569742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,8,128,1,float16,float16,0,0.11806399623552959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,2,128,1,float16,float16,0,0.6744106610616049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,2,128,1,float16,float16,0,0.05789333085219065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,32,128,1,float16,float16,0,0.0784693310658137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,1,128,1,float16,float16,0,0.05587733288606008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,4,128,1,float16,float16,0,0.06328533093134563
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,8,128,1,float16,float16,0,0.06502933303515117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,32,128,1,float16,float16,0,0.044549331068992615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,2,128,1,float16,float16,0,0.18766933679580688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,2,128,1,float16,float16,0,0.038746667404969536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,8,128,1,float16,float16,0,0.04151466737190882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,1,128,1,float16,float16,0,0.03786666691303253
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,4,128,1,float16,float16,0,0.04081066697835922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,8,128,1,float16,float16,0,0.7905386288960775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,1,128,1,float16,float16,0,0.8862559795379639
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,32,128,1,float16,float16,0,1.4923200607299805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,2,128,1,float16,float16,0,0.9897920290629069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,32,128,1,float16,float16,0,0.7524906794230143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,32,1,128,1,float16,float16,0,1.789632002512614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,4,128,1,float16,float16,0,1.1918400128682454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,32,2,128,1,float16,float16,0,1.983141263326009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,1,128,1,float16,float16,0,0.43703468640645343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,8,128,1,float16,float16,0,1.217402696609497
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,32,4,128,1,float16,float16,0,2.403098742167155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,4,128,1,float16,float16,0,0.5962186654408773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,1,128,1,float16,float16,0,0.2275786598523458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,2,128,1,float16,float16,0,0.49556267261505127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,8,128,1,float16,float16,0,0.6084479888280233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,32,128,1,float16,float16,0,0.38630398114522296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,32,8,128,1,float16,float16,0,2.4347893397013345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,2,128,1,float16,float16,0,0.2546666661898295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,32,128,1,float16,float16,0,0.20403200387954712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,8,128,1,float16,float16,0,0.31427733103434247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,1,128,1,float16,float16,0,0.12307199835777283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,4,128,1,float16,float16,0,0.1602773368358612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,32,128,1,float16,float16,0,0.10935466488202412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,4,128,1,float16,float16,0,0.08866133292516072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,8,128,1,float16,float16,0,0.09030933181444804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,32,128,1,float16,float16,0,0.06228266656398773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,1,128,1,float16,float16,0,0.07016533116499583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,2,128,1,float16,float16,0,0.13596799969673157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,2,128,1,float16,float16,0,0.07496533294518788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,4,128,1,float16,float16,0,0.3063093423843384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,1,128,1,float16,float16,0,0.04011733333269755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,32,128,1,float16,float16,0,0.032325332363446556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,2,128,1,float16,float16,0,0.041946664452552795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,4,128,1,float16,float16,0,0.04650133351484934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,1,128,1,float16,float16,0,0.026170666019121807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,8,128,1,float16,float16,0,0.04926399886608124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,32,128,1,float16,float16,0,0.023845332364241283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,8,128,1,float16,float16,0,0.02938133229811986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,4,128,1,float16,float16,0,0.029071999092896778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,2,128,1,float16,float16,0,0.021295999487241108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,1,128,1,float16,float16,0,0.021114667256673176
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,4,128,1,float16,float16,0,0.02293866624434789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,2,128,1,float16,float16,0,0.02720000098148982
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,8,128,1,float16,float16,0,0.022991999983787537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,8,128,1,float16,float16,0,0.16499200463294983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,1,128,1,float16,float16,0,0.33854401111602783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,2,128,1,float16,float16,0,0.3970133463541667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,4,128,1,float16,float16,0,0.5059093236923218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,32,1,128,1,float16,float16,0,0.7923306624094645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,32,128,1,float16,float16,0,0.36208001772562665
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,1,128,1,float16,float16,0,0.1778293251991272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,8,128,1,float16,float16,0,0.517685333887736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,8,128,1,float16,float16,0,0.26522133747736615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,32,2,128,1,float16,float16,0,0.8955893516540527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,32,4,128,1,float16,float16,0,1.1135466893513997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,32,8,128,1,float16,float16,0,1.1295733451843262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,2,128,1,float16,float16,0,0.20418665806452432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,32,128,1,float16,float16,0,0.19008000691731772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,4,128,1,float16,float16,0,0.2552746733029683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,8,128,1,float16,float16,0,0.13915200034777322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,32,128,1,float16,float16,0,0.10046933094660442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,32,128,1,float16,float16,0,0.7246879736582438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,2,128,1,float16,float16,0,0.060773332913716636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,1,128,1,float16,float16,0,0.055386667450269066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,4,128,1,float16,float16,0,0.07389333347479503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,32,128,1,float16,float16,0,0.05694933235645294
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,1,128,1,float16,float16,0,0.031152000029881794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,8,128,1,float16,float16,0,0.03845333307981491
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,4,128,1,float16,float16,0,0.036714665591716766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,2,128,1,float16,float16,0,0.032645332316557564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,1,128,1,float16,float16,0,0.09551999966303508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,32,128,1,float16,float16,0,0.027642667293548584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,2,128,1,float16,float16,0,0.02274133265018463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,1,128,1,float16,float16,0,0.021205333371957142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,4,128,1,float16,float16,0,0.024522667129834492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,8,128,1,float16,float16,0,0.024906667570273083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,1,128,1,float16,float16,0,0.01580799991885821
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,32,128,1,float16,float16,0,0.01905599981546402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,2,128,1,float16,float16,0,0.10893332958221436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,2,128,1,float16,float16,0,0.01632000009218852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,4,128,1,float16,float16,0,0.018016000588734944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,8,128,1,float16,float16,0,0.01798933371901512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,32,128,1,float16,float16,0,0.013877333452304205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,2,128,1,float16,float16,0,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,4,128,1,float16,float16,0,0.01312000056107839
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,8,128,1,float16,float16,0,0.012874666601419449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,4,128,1,float16,float16,0,0.13313066959381104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,1,128,1,float16,float16,0,0.012693333129088083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,1,128,1,float16,float16,0,0.17428267002105713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,8,128,1,float16,float16,0,0.0777706652879715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,2,128,1,float16,float16,0,0.1997119983037313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,32,128,1,float16,float16,0,0.3564106623331706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,32,2,128,1,float16,float16,0,0.3867573340733846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,32,4,128,1,float16,float16,0,0.49060265223185223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,32,8,128,1,float16,float16,0,0.5103573401769003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,32,1,128,1,float16,float16,0,0.3335253397623698
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,32,128,1,float16,float16,0,0.18662399053573608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,4,128,1,float16,float16,0,0.25440533955891925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,2,128,1,float16,float16,0,0.1095413366953532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,1,128,1,float16,float16,0,0.09723200400670369
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,8,128,1,float16,float16,0,0.2623680035273234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,2,128,1,float16,float16,0,0.06144000093142191
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,1,128,1,float16,float16,0,0.05682133138179779
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,32,128,1,float16,float16,0,0.09904000163078308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,8,128,1,float16,float16,0,0.07713599999745686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,4,128,1,float16,float16,0,0.07400533556938171
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,4,128,1,float16,float16,0,0.1334933340549469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,1,128,1,float16,float16,0,0.03009066730737686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,2,128,1,float16,float16,0,0.031871999303499855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,8,128,1,float16,float16,0,0.1376479963461558
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,32,128,1,float16,float16,0,0.02788266787926356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,8,128,1,float16,float16,0,0.03742400060097376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,2,128,1,float16,float16,0,0.02237333357334137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,4,128,1,float16,float16,0,0.024277334411938984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,1,128,1,float16,float16,0,0.02089600016673406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,32,128,1,float16,float16,0,0.057029331723848976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,4,128,1,float16,float16,0,0.03583999971548716
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,32,128,1,float16,float16,0,0.01868266612291336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,1,128,1,float16,float16,0,0.015589332828919092
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,2,128,1,float16,float16,0,0.015813333292802174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,8,128,1,float16,float16,0,0.024533333877722423
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,4,128,1,float16,float16,0,0.017594666530688603
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,8,128,1,float16,float16,0,0.017866666118303936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,4,128,1,float16,float16,0,0.012965332716703415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,1,128,1,float16,float16,0,0.012506666282812754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,1,128,1,float16,float16,0,0.01190399999419848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,8,128,1,float16,float16,0,0.013093333691358566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,32,128,1,float16,float16,0,0.013663999736309052
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,32,128,1,float16,float16,0,0.012453333785136541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,2,128,1,float16,float16,0,0.012362666428089142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,2,128,1,float16,float16,0,0.01209066684047381
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,4,128,1,float16,float16,0,0.012229333321253458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,8,128,1,float16,float16,0,0.012186666329701742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,1,128,1,float16,float16,0,0.15941333770751953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,2,128,1,float16,float16,0,0.17208532492319742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,4,128,1,float16,float16,0,0.19741866985956827
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,32,128,1,float16,float16,0,0.1220746636390686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,32,8,128,1,float16,float16,0,0.38329601287841797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,32,128,1,float16,float16,0,0.23409066597620645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,1,128,1,float16,float16,0,0.08762666583061218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,8,128,1,float16,float16,0,0.1986080010732015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,32,4,128,1,float16,float16,0,0.3815679947535197
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,32,1,128,1,float16,float16,0,0.30803199609120685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,32,2,128,1,float16,float16,0,0.3304213285446167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,2,128,1,float16,float16,0,0.09195199608802795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,8,128,1,float16,float16,0,0.10784533619880676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,32,128,1,float16,float16,0,0.03455466777086258
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,1,128,1,float16,float16,0,0.04351999859015147
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,2,128,1,float16,float16,0,0.02770666778087616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,32,128,1,float16,float16,0,0.06897599995136261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,1,128,1,float16,float16,0,0.02630399912595749
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,4,128,1,float16,float16,0,0.0496319979429245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,4,128,1,float16,float16,0,0.10466667016347249
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,4,128,1,float16,float16,0,0.029818666477998097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,2,128,1,float16,float16,0,0.045269335309664406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,8,128,1,float16,float16,0,0.054858664671579994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,8,128,1,float16,float16,0,0.030063999195893604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,1,128,1,float16,float16,0,0.019023999571800232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,32,128,1,float16,float16,0,0.022250667214393616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,2,128,1,float16,float16,0,0.019205333044131596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,4,128,1,float16,float16,0,0.021136000752449036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,8,128,1,float16,float16,0,0.021253332495689392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,1,128,1,float16,float16,0,0.014368000129858652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,2,128,1,float16,float16,0,0.014709333578745524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,32,128,1,float16,float16,0,0.01249066616098086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,1,128,1,float16,float16,0,0.011770666887362799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,8,128,1,float16,float16,0,0.015034666905800501
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,32,128,1,float16,float16,0,0.01573866605758667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,2,128,1,float16,float16,0,0.011663999408483505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,4,128,1,float16,float16,0,0.011968000481526056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,8,128,1,float16,float16,0,0.011978667229413986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,4,128,1,float16,float16,0,0.015114666273196539
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,1,128,1,float16,float16,0,0.011503999431928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,32,128,1,float16,float16,0,0.0122079998254776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,8,128,1,float16,float16,0,0.011823999385039011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,4,128,1,float16,float16,0,0.011695999652147293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,2,128,1,float16,float16,0,0.01146666705608368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,1,128,1,float16,float16,0,0.14937067031860352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,32,128,1,float16,float16,0,0.1853813330332438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,2,128,1,float16,float16,0,0.1557973325252533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,32,1,128,1,float16,float16,0,0.29130132993062335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,32,4,128,1,float16,float16,0,0.327674667040507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,32,8,128,1,float16,float16,0,0.33299734195073444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,32,2,128,1,float16,float16,0,0.3054080009460449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,32,128,1,float16,float16,0,0.09969066580136617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,4,128,1,float16,float16,0,0.16809600591659546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,1,128,1,float16,float16,0,0.07470400134722392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,8,128,1,float16,float16,0,0.17139200369517008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,32,128,1,float16,float16,0,0.04925866425037384
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,1,128,1,float16,float16,0,0.041178666055202484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,2,128,1,float16,float16,0,0.07644799848397572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,4,128,1,float16,float16,0,0.0811413327852885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,8,128,1,float16,float16,0,0.087226668993632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,32,128,1,float16,float16,0,0.02887466549873352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,2,128,1,float16,float16,0,0.0429013321797053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,4,128,1,float16,float16,0,0.04487466812133789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,1,128,1,float16,float16,0,0.02521066615978877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,8,128,1,float16,float16,0,0.04509866734345754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,2,128,1,float16,float16,0,0.02586666742960612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,8,128,1,float16,float16,0,0.027669332921504974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,32,128,1,float16,float16,0,0.0199946661790212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,4,128,1,float16,float16,0,0.02749866743882497
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,2,128,1,float16,float16,0,0.01851733277241389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,1,128,1,float16,float16,0,0.018592000007629395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,32,128,1,float16,float16,0,0.014922666052977243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,8,128,1,float16,float16,0,0.019109333554903667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,4,128,1,float16,float16,0,0.019205333044131596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,1,128,1,float16,float16,0,0.01441066712141037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,2,128,1,float16,float16,0,0.014293332894643148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,4,128,1,float16,float16,0,0.014373333503802618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,32,128,1,float16,float16,0,0.012154666086037954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,8,128,1,float16,float16,0,0.014592000593741735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,1,128,1,float16,float16,0,0.011616000284751257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,4,128,1,float16,float16,0,0.011973333855470022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,8,128,1,float16,float16,0,0.011829332758982977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,32,128,1,float16,float16,0,0.01201066623131434
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,1,128,1,float16,float16,0,0.01139733319481214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,2,128,1,float16,float16,0,0.01166933278242747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,4,128,1,float16,float16,0,0.011642667154471079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,8,128,1,float16,float16,0,0.011717333147923151
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,0,0.016373333831628162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,0,0.02938133229811986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,32,128,1,float16,float16,0,0.130703995625178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,0,0.036090667049090065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,0,0.05831466615200043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,0,0.021482666333516438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,32,128,1,float16,float16,0,0.06964266796906789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,0,0.0408746674656868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,0,0.08877333005269368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,0,0.013642666240533194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,0,0.05190933247407278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,32,128,1,float16,float16,0,0.04049599915742874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,0,0.02298133323589961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,0,0.013248000293970108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,0,0.019797333826621372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,0,0.02258666604757309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,0,0.02716800073782603
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,0,0.033439998825391136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,32,128,1,float16,float16,0,0.02446399877468745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,0,0.027077332139015198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,0,0.0129120002190272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,0,0.019567999988794327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,0,0.01959466685851415
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,32,128,1,float16,float16,0,0.018063999712467194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,0,0.022277332842350006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,0,0.012602667013804117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,0,0.01926933353145917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,0,0.019167999426523846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,0,0.01932799940307935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,32,128,1,float16,float16,0,0.01403733342885971
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,0,0.018170667191346485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,0,0.012602667013804117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,32,128,1,float16,float16,0,0.011359999577204386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,0,0.018570666511853535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,0,0.018725333114465077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,0,0.01250133290886879
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,0,0.01828266680240631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,0,0.01841066653529803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,0,0.018133333573738735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,32,128,1,float16,float16,0,0.011237333218256632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,0,0.012421333541472753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,0,0.012400000045696894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,0,0.012229333321253458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,0,0.012527999778588613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,1,128,1,float16,float16,0,36.452868143717446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,24,128,1,float16,float16,0,37.47430928548177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,2,128,1,float16,float16,0,37.03797912597656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,4,128,1,float16,float16,0,37.1380360921224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,1,128,1,float16,float16,0,18.570772806803387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,24,128,1,float16,float16,0,18.89837392171224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,24,2,128,1,float16,float16,0,73.8610127766927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,24,1,128,1,float16,float16,0,74.58515930175781
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,24,4,128,1,float16,float16,0,74.83744303385417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,2,128,1,float16,float16,0,18.483776092529297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,24,8,128,1,float16,float16,0,74.91027323404948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,8,128,1,float16,float16,0,37.14749399820963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,4,128,1,float16,float16,0,19.059594472249348
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,24,128,1,float16,float16,0,9.623839696248373
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,1,128,1,float16,float16,0,9.392309188842773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,8,128,1,float16,float16,0,18.90511957804362
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,2,128,1,float16,float16,0,9.678421020507812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,4,128,1,float16,float16,0,9.28869883219401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,8,128,1,float16,float16,0,9.607413609822592
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,1,128,1,float16,float16,0,20.967077891031902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,2,128,1,float16,float16,0,20.912986755371094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,24,128,1,float16,float16,0,21.758773803710938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,4,128,1,float16,float16,0,21.396901448567707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,1,128,1,float16,float16,0,10.486442565917969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,24,128,1,float16,float16,0,10.734213511149088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,2,128,1,float16,float16,0,10.557029088338217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,24,1,128,1,float16,float16,0,41.30170186360677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,24,2,128,1,float16,float16,0,41.9964853922526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,24,4,128,1,float16,float16,0,42.72377014160156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,24,8,128,1,float16,float16,0,43.32477823893229
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,8,128,1,float16,float16,0,21.51110331217448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,24,128,1,float16,float16,0,5.3678131103515625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,1,128,1,float16,float16,0,5.100346565246582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,2,128,1,float16,float16,0,5.283637364705403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,4,128,1,float16,float16,0,5.412799835205078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,4,128,1,float16,float16,0,10.575263977050781
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,8,128,1,float16,float16,0,10.477120081583658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,8,128,1,float16,float16,0,5.37995719909668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,1,128,1,float16,float16,0,14.697402954101562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,2,128,1,float16,float16,0,14.62719472249349
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,24,128,1,float16,float16,0,15.260757446289062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,4,128,1,float16,float16,0,15.24777094523112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,1,128,1,float16,float16,0,7.313077290852864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,24,128,1,float16,float16,0,7.546586354573567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,24,1,128,1,float16,float16,0,29.4434814453125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,2,128,1,float16,float16,0,7.49722162882487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,24,2,128,1,float16,float16,0,29.569595336914062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,24,4,128,1,float16,float16,0,29.76928965250651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,24,8,128,1,float16,float16,0,30.06897481282552
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,1,128,1,float16,float16,0,3.6681121190389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,24,128,1,float16,float16,0,3.9174718856811523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,8,128,1,float16,float16,0,15.19757334391276
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,2,128,1,float16,float16,0,3.770170529683431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,4,128,1,float16,float16,0,3.8019946416219077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,4,128,1,float16,float16,0,7.4750722249348955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,8,128,1,float16,float16,0,7.350810368855794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,8,128,1,float16,float16,0,3.7481492360432944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,1,128,1,float16,float16,0,18.975935618082683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,2,128,1,float16,float16,0,18.921695709228516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,24,128,1,float16,float16,0,19.668655395507812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,4,128,1,float16,float16,0,19.215328216552734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,1,128,1,float16,float16,0,9.266693115234375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,24,128,1,float16,float16,0,9.81274668375651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,2,128,1,float16,float16,0,9.533434549967447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,24,1,128,1,float16,float16,0,38.4264170328776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,24,2,128,1,float16,float16,0,38.221972147623696
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,24,128,1,float16,float16,0,4.93015988667806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,24,4,128,1,float16,float16,0,38.98681131998698
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,24,8,128,1,float16,float16,0,39.41740163167318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,8,128,1,float16,float16,0,19.608895619710285
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,4,128,1,float16,float16,0,9.60595703125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,8,128,1,float16,float16,0,9.667226791381836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,1,128,1,float16,float16,0,4.704064051310222
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,2,128,1,float16,float16,0,4.755546569824219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,24,128,1,float16,float16,0,2.544688065846761
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,1,128,1,float16,float16,0,2.4322667121887207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,4,128,1,float16,float16,0,4.811893463134766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,8,128,1,float16,float16,0,4.907519976298015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,2,128,1,float16,float16,0,2.392805258433024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,4,128,1,float16,float16,0,2.459551970163981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,8,128,1,float16,float16,0,2.5055360794067383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,1,128,1,float16,float16,0,10.673355102539062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,2,128,1,float16,float16,0,10.818986256917318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,24,128,1,float16,float16,0,11.549568176269531
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,4,128,1,float16,float16,0,11.468138376871744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,2,128,1,float16,float16,0,5.437215805053711
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,1,128,1,float16,float16,0,5.323637326558431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,24,128,1,float16,float16,0,5.622778574625651
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,24,1,128,1,float16,float16,0,21.639567057291668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,24,2,128,1,float16,float16,0,21.974756876627605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,24,4,128,1,float16,float16,0,22.508084615071613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,24,8,128,1,float16,float16,0,22.62079366048177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,8,128,1,float16,float16,0,11.172618865966797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,24,128,1,float16,float16,0,2.904197374979655
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,1,128,1,float16,float16,0,2.6755040486653647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,2,128,1,float16,float16,0,2.719381332397461
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,4,128,1,float16,float16,0,5.61076291402181
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,8,128,1,float16,float16,0,5.545674641927083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,24,128,1,float16,float16,0,1.4740853309631348
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,1,128,1,float16,float16,0,1.387466589609782
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,4,128,1,float16,float16,0,2.75216007232666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,8,128,1,float16,float16,0,2.808842658996582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,2,128,1,float16,float16,0,1.4368480046590169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,4,128,1,float16,float16,0,1.5065120061238606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,8,128,1,float16,float16,0,1.4699573516845703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,1,128,1,float16,float16,0,9.987466812133789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,2,128,1,float16,float16,0,9.984954833984375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,24,128,1,float16,float16,0,10.76419194539388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,1,128,1,float16,float16,0,4.856538772583008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,24,128,1,float16,float16,0,5.295749346415202
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,24,1,128,1,float16,float16,0,20.0314458211263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,24,2,128,1,float16,float16,0,20.160357157389324
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,24,4,128,1,float16,float16,0,21.01260248819987
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,24,8,128,1,float16,float16,0,20.651504516601562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,4,128,1,float16,float16,0,10.413920084635416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,8,128,1,float16,float16,0,10.582821528116861
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,24,128,1,float16,float16,0,2.684330622355143
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,2,128,1,float16,float16,0,4.961189270019531
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,4,128,1,float16,float16,0,5.054656028747559
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,1,128,1,float16,float16,0,2.3923306465148926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,2,128,1,float16,float16,0,2.455029328664144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,24,128,1,float16,float16,0,1.3607360521952312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,1,128,1,float16,float16,0,1.246997356414795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,2,128,1,float16,float16,0,1.2724640369415283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,8,128,1,float16,float16,0,5.193696022033691
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,8,128,1,float16,float16,0,2.6356852849324546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,24,128,1,float16,float16,0,0.7205546696980795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,1,128,1,float16,float16,0,0.6502986749013265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,4,128,1,float16,float16,0,2.5567893981933594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,8,128,1,float16,float16,0,1.3100693225860596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,4,128,1,float16,float16,0,1.2992586294809978
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,2,128,1,float16,float16,0,0.6713279883066813
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,4,128,1,float16,float16,0,0.7146186828613281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,8,128,1,float16,float16,0,0.7262933254241943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,2,128,1,float16,float16,0,5.780693054199219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,24,128,1,float16,float16,0,6.225781122843425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,4,128,1,float16,float16,0,6.097077051798503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,1,128,1,float16,float16,0,5.5425974527994795
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,1,128,1,float16,float16,0,2.810373306274414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,2,128,1,float16,float16,0,2.8783305486043296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,24,2,128,1,float16,float16,0,11.729131062825521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,24,128,1,float16,float16,0,3.0706345240275064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,24,1,128,1,float16,float16,0,11.593674977620443
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,24,8,128,1,float16,float16,0,12.232203165690104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,24,4,128,1,float16,float16,0,12.285690307617188
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,8,128,1,float16,float16,0,6.137162526448567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,1,128,1,float16,float16,0,1.4312960306803386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,24,128,1,float16,float16,0,1.5904639561971028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,2,128,1,float16,float16,0,1.4570773442586262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,4,128,1,float16,float16,0,1.5603466033935547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,4,128,1,float16,float16,0,2.939530690511068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,24,128,1,float16,float16,0,0.8422133127848307
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,1,128,1,float16,float16,0,0.7402559916178385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,8,128,1,float16,float16,0,2.9512853622436523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,8,128,1,float16,float16,0,1.5337173144022624
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,2,128,1,float16,float16,0,0.742186705271403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,4,128,1,float16,float16,0,0.7916159629821777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,24,128,1,float16,float16,0,0.461082657178243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,1,128,1,float16,float16,0,0.39986666043599445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,2,128,1,float16,float16,0,0.4091413418451945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,8,128,1,float16,float16,0,0.8014026482899984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,4,128,1,float16,float16,0,0.44050665696461994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,8,128,1,float16,float16,0,0.4253386656443278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,1,128,1,float16,float16,0,5.159296035766602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,24,128,1,float16,float16,0,6.1861012776692705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,2,128,1,float16,float16,0,5.3809388478597
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,4,128,1,float16,float16,0,5.7697709401448565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,1,128,1,float16,float16,0,2.5785120328267417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,24,128,1,float16,float16,0,3.0326401392618814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,2,128,1,float16,float16,0,2.6340959866841636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,24,4,128,1,float16,float16,0,12.078266143798828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,24,8,128,1,float16,float16,0,12.115408579508463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,24,2,128,1,float16,float16,0,11.232816060384115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,8,128,1,float16,float16,0,5.81223996480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,24,1,128,1,float16,float16,0,10.825519561767578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,24,128,1,float16,float16,0,1.5295519828796387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,4,128,1,float16,float16,0,2.8278185526529946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,8,128,1,float16,float16,0,2.903248151143392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,1,128,1,float16,float16,0,1.3572479883829753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,2,128,1,float16,float16,0,1.4140853881835938
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,8,128,1,float16,float16,0,1.4848480224609375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,4,128,1,float16,float16,0,0.747221310933431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,2,128,1,float16,float16,0,0.6960852940877279
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,1,128,1,float16,float16,0,0.662448008855184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,24,128,1,float16,float16,0,0.42527465025583905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,1,128,1,float16,float16,0,0.35734931627909344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,8,128,1,float16,float16,0,0.7638400395711263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,2,128,1,float16,float16,0,0.36602667967478436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,4,128,1,float16,float16,0,0.3914399941762288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,24,128,1,float16,float16,0,0.2327786684036255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,1,128,1,float16,float16,0,0.20234666268030801
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,2,128,1,float16,float16,0,0.2081813414891561
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,4,128,1,float16,float16,0,1.4670400619506836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,24,128,1,float16,float16,0,0.8020266691843668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,8,128,1,float16,float16,0,0.3994346857070923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,4,128,1,float16,float16,0,0.22181334098180136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,8,128,1,float16,float16,0,0.22581332921981812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,1,128,1,float16,float16,0,2.968501408894857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,24,128,1,float16,float16,0,3.6706453959147134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,2,128,1,float16,float16,0,3.1573813756306968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,24,4,128,1,float16,float16,0,7.121045430501302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,24,1,128,1,float16,float16,0,5.861696243286133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,24,8,128,1,float16,float16,0,7.205493291219075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,24,128,1,float16,float16,0,1.8764853477478027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,24,2,128,1,float16,float16,0,6.221162796020508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,4,128,1,float16,float16,0,3.4626026153564453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,8,128,1,float16,float16,0,3.4863198598225913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,1,128,1,float16,float16,0,0.7723519802093506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,4,128,1,float16,float16,0,1.731093406677246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,8,128,1,float16,float16,0,1.7431626319885254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,1,128,1,float16,float16,0,1.4928107261657715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,2,128,1,float16,float16,0,1.5703946749369304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,2,128,1,float16,float16,0,0.80841064453125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,24,128,1,float16,float16,0,0.9643146991729736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,24,128,1,float16,float16,0,0.4981386661529541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,4,128,1,float16,float16,0,0.8842986424763998
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,1,128,1,float16,float16,0,0.4033493200937907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,8,128,1,float16,float16,0,0.8879573345184326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,2,128,1,float16,float16,0,0.42215466499328613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,24,128,1,float16,float16,0,0.26679466168085736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,2,128,1,float16,float16,0,0.23177067438761392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,8,128,1,float16,float16,0,0.4718613227208455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,1,128,1,float16,float16,0,0.22150399287541708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,4,128,1,float16,float16,0,0.4639306863149007
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,8,128,1,float16,float16,0,0.2491733431816101
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,4,128,1,float16,float16,0,0.24968532721201578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,1,128,1,float16,float16,0,0.12499200304349263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,4,128,1,float16,float16,0,0.13571199774742126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,2,128,1,float16,float16,0,0.1288159986337026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,8,128,1,float16,float16,0,0.14230933785438538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,24,128,1,float16,float16,0,0.15145599842071533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,1,128,1,float16,float16,0,2.9334774017333984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,24,128,1,float16,float16,0,3.928405443827311
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,2,128,1,float16,float16,0,3.1073439915974936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,24,128,1,float16,float16,0,2.0049546559651694
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,1,128,1,float16,float16,0,1.4772052764892578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,24,8,128,1,float16,float16,0,7.479114532470703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,24,2,128,1,float16,float16,0,6.40015983581543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,24,1,128,1,float16,float16,0,6.0445601145426435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,4,128,1,float16,float16,0,3.512533187866211
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,8,128,1,float16,float16,0,3.5834293365478516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,2,128,1,float16,float16,0,1.608693281809489
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,4,128,1,float16,float16,0,1.7759626706441243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,24,4,128,1,float16,float16,0,6.951498667399089
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,8,128,1,float16,float16,0,1.856298605600993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,4,128,1,float16,float16,0,0.9105066458384196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,2,128,1,float16,float16,0,0.41757333278656006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,1,128,1,float16,float16,0,0.7626346747080485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,4,128,1,float16,float16,0,0.4668266773223877
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,8,128,1,float16,float16,0,0.9149813652038574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,24,128,1,float16,float16,0,0.5066293478012085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,8,128,1,float16,float16,0,0.47100265820821124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,2,128,1,float16,float16,0,0.7958986759185791
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,24,128,1,float16,float16,0,0.9896426995595297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,24,128,1,float16,float16,0,0.2729439934094747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,1,128,1,float16,float16,0,0.21013333400090536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,2,128,1,float16,float16,0,0.22267200549443564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,8,128,1,float16,float16,0,0.25089067220687866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,4,128,1,float16,float16,0,0.2458933393160502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,4,128,1,float16,float16,0,0.13913066188494363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,24,128,1,float16,float16,0,0.09128533800443013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,2,128,1,float16,float16,0,0.12442133824030559
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,24,128,1,float16,float16,0,0.15010133385658264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,8,128,1,float16,float16,0,0.14109866817792258
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,1,128,1,float16,float16,0,0.11880532900492351
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,1,128,1,float16,float16,0,0.07221866647402446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,2,128,1,float16,float16,0,0.07627200086911519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,4,128,1,float16,float16,0,0.07809600234031677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,8,128,1,float16,float16,0,0.08028266827265422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,1,128,1,float16,float16,0,0.3856533368428548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,24,128,1,float16,float16,0,2.8278986612955728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,1,128,1,float16,float16,0,1.877343972524007
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,2,128,1,float16,float16,0,2.075040022532145
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,4,128,1,float16,float16,0,2.4872639973958335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,24,128,1,float16,float16,0,1.424623966217041
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,1,128,1,float16,float16,0,0.9474240144093832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,24,1,128,1,float16,float16,0,3.7829065322875977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,24,2,128,1,float16,float16,0,4.160778681437175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,4,128,1,float16,float16,0,1.2541653315226238
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,8,128,1,float16,float16,0,2.5574399630228677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,24,128,1,float16,float16,0,0.7407146294911703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,2,128,1,float16,float16,0,1.046847979227702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,8,128,1,float16,float16,0,1.2764533360799153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,24,128,1,float16,float16,0,0.3743520180384318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,2,128,1,float16,float16,0,0.5306986570358276
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,1,128,1,float16,float16,0,0.4824639956156413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,24,4,128,1,float16,float16,0,4.954757372538249
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,2,128,1,float16,float16,0,0.27752532561620075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,4,128,1,float16,float16,0,0.3288266658782959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,8,128,1,float16,float16,0,0.6547840038935343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,1,128,1,float16,float16,0,0.2523413300514221
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,8,128,1,float16,float16,0,0.3375306526819865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,1,128,1,float16,float16,0,0.13666666547457376
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,4,128,1,float16,float16,0,0.1737920045852661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,24,128,1,float16,float16,0,0.11086933811505635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,8,128,1,float16,float16,0,0.17991467316945395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,1,128,1,float16,float16,0,0.07631466786066692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,2,128,1,float16,float16,0,0.14988799889882407
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,4,128,1,float16,float16,0,0.640341321627299
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,2,128,1,float16,float16,0,0.08232533435026805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,4,128,1,float16,float16,0,0.09660266836484273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,8,128,1,float16,float16,0,0.09850666920344035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,24,128,1,float16,float16,0,0.06541333099206288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,2,128,1,float16,float16,0,0.04975466430187225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,1,128,1,float16,float16,0,0.04790399968624115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,4,128,1,float16,float16,0,0.05306666592756907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,8,128,1,float16,float16,0,0.05403733253479004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,1,128,1,float16,float16,0,0.027215999861558277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,24,128,1,float16,float16,0,0.03229333211978277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,24,128,1,float16,float16,0,0.20085332791010538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,4,128,1,float16,float16,0,0.030847998956839245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,2,128,1,float16,float16,0,0.028815999627113342
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,8,128,1,float16,float16,0,0.030943999687830608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,24,8,128,1,float16,float16,0,5.1102345784505205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,1,128,1,float16,float16,0,0.685749371846517
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,24,128,1,float16,float16,0,1.1775786876678467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,24,1,128,1,float16,float16,0,1.3793066342671711
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,24,2,128,1,float16,float16,0,1.573845386505127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,24,4,128,1,float16,float16,0,1.9902292887369792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,4,128,1,float16,float16,0,0.9915680090586344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,24,128,1,float16,float16,0,0.5918346643447876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,24,8,128,1,float16,float16,0,2.011493364969889
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,2,128,1,float16,float16,0,0.794597307840983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,24,128,1,float16,float16,0,0.3057600061098735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,8,128,1,float16,float16,0,0.5127520163853964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,1,128,1,float16,float16,0,0.3407893180847168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,8,128,1,float16,float16,0,1.0267786979675293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,2,128,1,float16,float16,0,0.3935786485671997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,2,128,1,float16,float16,0,0.20633065700531006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,4,128,1,float16,float16,0,0.2579573392868042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,4,128,1,float16,float16,0,0.5000053246816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,2,128,1,float16,float16,0,0.11157332857449849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,24,128,1,float16,float16,0,0.16422933340072632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,4,128,1,float16,float16,0,0.13640532890955606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,1,128,1,float16,float16,0,0.05494933327039083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,8,128,1,float16,float16,0,0.14007467031478882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,8,128,1,float16,float16,0,0.26554133494695026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,2,128,1,float16,float16,0,0.06122666597366333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,4,128,1,float16,float16,0,0.07505600154399872
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,8,128,1,float16,float16,0,0.07819733520348866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,24,128,1,float16,float16,0,0.050426666935284935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,4,128,1,float16,float16,0,0.03842133283615112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,24,128,1,float16,float16,0,0.09079999725023906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,2,128,1,float16,float16,0,0.034671999514102936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,8,128,1,float16,float16,0,0.03885333240032196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,1,128,1,float16,float16,0,0.03329599897066752
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,24,128,1,float16,float16,0,0.025834667185942333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,4,128,1,float16,float16,0,0.024800000091393787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,1,128,1,float16,float16,0,0.021418665846188862
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,1,128,1,float16,float16,0,0.016208000481128693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,1,128,1,float16,float16,0,0.09913067022959392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,8,128,1,float16,float16,0,0.024869332710901897
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,24,128,1,float16,float16,0,0.018464000274737675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,2,128,1,float16,float16,0,0.0234400009115537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,1,128,1,float16,float16,0,0.17894933621088663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,2,128,1,float16,float16,0,0.016293333222468693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,4,128,1,float16,float16,0,0.01823466643691063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,8,128,1,float16,float16,0,0.018191999445358913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,4,128,1,float16,float16,0,0.4308160146077474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,2,128,1,float16,float16,0,0.3269600073496501
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,1,128,1,float16,float16,0,0.26632533470789593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,1,128,1,float16,float16,0,0.13983466227849325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,24,128,1,float16,float16,0,0.28862933317820233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,8,128,1,float16,float16,0,0.44946666558583576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,2,128,1,float16,float16,0,0.1666719913482666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,24,1,128,1,float16,float16,0,0.6158133347829183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,24,4,128,1,float16,float16,0,0.926042636235555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,24,2,128,1,float16,float16,0,0.7095306714375814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,24,128,1,float16,float16,0,0.5689866542816162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,24,128,1,float16,float16,0,0.14918933312098184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,8,128,1,float16,float16,0,0.23055466016133627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,24,8,128,1,float16,float16,0,0.953333298365275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,24,128,1,float16,float16,0,0.0836853285630544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,4,128,1,float16,float16,0,0.22437334060668945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,1,128,1,float16,float16,0,0.043893332282702126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,24,128,1,float16,float16,0,0.04481600224971771
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,4,128,1,float16,float16,0,0.06563733518123627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,8,128,1,float16,float16,0,0.06943466762701671
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,2,128,1,float16,float16,0,0.028245332340399425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,1,128,1,float16,float16,0,0.026389333109060924
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,4,128,1,float16,float16,0,0.03170666595300039
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,8,128,1,float16,float16,0,0.03253333270549774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,24,128,1,float16,float16,0,0.02072000006834666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,1,128,1,float16,float16,0,0.0788266658782959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,2,128,1,float16,float16,0,0.0906986693541209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,1,128,1,float16,float16,0,0.016714667280515034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,4,128,1,float16,float16,0,0.11586133639017741
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,4,128,1,float16,float16,0,0.0198186660806338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,2,128,1,float16,float16,0,0.017952000101407368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,1,128,1,float16,float16,0,0.013365333278973898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,8,128,1,float16,float16,0,0.019999999552965164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,24,128,1,float16,float16,0,0.015728000551462173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,4,128,1,float16,float16,0,0.015274666249752045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,2,128,1,float16,float16,0,0.013536000003417334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,8,128,1,float16,float16,0,0.015487999965747198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,24,128,1,float16,float16,0,0.01351999988158544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,4,128,1,float16,float16,0,0.012986666212479273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,2,128,1,float16,float16,0,0.012389333297808966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,1,128,1,float16,float16,0,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,8,128,1,float16,float16,0,0.012890666723251343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,8,128,1,float16,float16,0,0.12221333384513855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,2,128,1,float16,float16,0,0.049653331438700356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,1,128,1,float16,float16,0,0.13979199528694153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,2,128,1,float16,float16,0,0.16478400429089865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,24,128,1,float16,float16,0,0.28650132815043133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,24,1,128,1,float16,float16,0,0.26173333326975506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,24,2,128,1,float16,float16,0,0.31655999024709064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,4,128,1,float16,float16,0,0.2174933354059855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,8,128,1,float16,float16,0,0.22671999533971152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,24,128,1,float16,float16,0,0.14718400438626608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,24,4,128,1,float16,float16,0,0.4241386651992798
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,1,128,1,float16,float16,0,0.07890666524569194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,4,128,1,float16,float16,0,0.11567466457684834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,24,128,1,float16,float16,0,0.0817333310842514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,2,128,1,float16,float16,0,0.09088533123334248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,8,128,1,float16,float16,0,0.11975466211636861
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,2,128,1,float16,float16,0,0.04946133494377136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,4,128,1,float16,float16,0,0.06419733166694641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,24,8,128,1,float16,float16,0,0.4500853220621745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,24,128,1,float16,float16,0,0.04498666524887085
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,2,128,1,float16,float16,0,0.027445333699385326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,8,128,1,float16,float16,0,0.06850133339564006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,4,128,1,float16,float16,0,0.031285333136717476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,1,128,1,float16,float16,0,0.04312533140182495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,1,128,1,float16,float16,0,0.02532800038655599
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,1,128,1,float16,float16,0,0.016074666132529575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,4,128,1,float16,float16,0,0.01961600035429001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,24,128,1,float16,float16,0,0.020501332978407543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,8,128,1,float16,float16,0,0.019946667055288952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,24,128,1,float16,float16,0,0.015578666081031164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,2,128,1,float16,float16,0,0.01783466711640358
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,2,128,1,float16,float16,0,0.013183999806642532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,1,128,1,float16,float16,0,0.01292266696691513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,4,128,1,float16,float16,0,0.014906667172908783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,4,128,1,float16,float16,0,0.012703999876976013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,24,128,1,float16,float16,0,0.013194666554530462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,2,128,1,float16,float16,0,0.012389333297808966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,1,128,1,float16,float16,0,0.01249066616098086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,8,128,1,float16,float16,0,0.01516266663869222
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,8,128,1,float16,float16,0,0.031930667658646904
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,24,128,1,float16,float16,0,0.012213333199421564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,1,128,1,float16,float16,0,0.011952000359694162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,8,128,1,float16,float16,0,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,2,128,1,float16,float16,0,0.011920000116030375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,4,128,1,float16,float16,0,0.012144000579913458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,8,128,1,float16,float16,0,0.012138667205969492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,1,128,1,float16,float16,0,0.12652267018953958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,2,128,1,float16,float16,0,0.13822933038075766
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,4,128,1,float16,float16,0,0.16201066970825195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,24,128,1,float16,float16,0,0.18638932704925537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,24,2,128,1,float16,float16,0,0.2624906698862712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,24,4,128,1,float16,float16,0,0.31124265988667804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,8,128,1,float16,float16,0,0.16485333442687988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,24,8,128,1,float16,float16,0,0.3160586754480998
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,24,128,1,float16,float16,0,0.1006666620572408
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,24,1,128,1,float16,float16,0,0.2394239902496338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,1,128,1,float16,float16,0,0.06803733110427856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,4,128,1,float16,float16,0,0.04258666435877482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,24,128,1,float16,float16,0,0.05481599768002828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,24,128,1,float16,float16,0,0.027514666318893433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,2,128,1,float16,float16,0,0.03832533210515976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,4,128,1,float16,float16,0,0.08745599786440532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,8,128,1,float16,float16,0,0.04363733530044556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,8,128,1,float16,float16,0,0.09018666545550029
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,2,128,1,float16,float16,0,0.07409599920113881
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,4,128,1,float16,float16,0,0.025749333202838898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,2,128,1,float16,float16,0,0.02363733450571696
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,8,128,1,float16,float16,0,0.025861332813898723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,24,128,1,float16,float16,0,0.01746133342385292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,1,128,1,float16,float16,0,0.02216533323129018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,1,128,1,float16,float16,0,0.03638399889071783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,1,128,1,float16,float16,0,0.01469333345691363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,4,128,1,float16,float16,0,0.017157333592573803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,2,128,1,float16,float16,0,0.015040000279744467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,24,128,1,float16,float16,0,0.013290667285521826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,8,128,1,float16,float16,0,0.016778666526079178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,8,128,1,float16,float16,0,0.012576000144084295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,4,128,1,float16,float16,0,0.012613333761692047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,2,128,1,float16,float16,0,0.012256000190973282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,1,128,1,float16,float16,0,0.012170666207869848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,24,128,1,float16,float16,0,0.012170666207869848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,2,128,1,float16,float16,0,0.01173866664369901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,8,128,1,float16,float16,0,0.011781333635250727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,4,128,1,float16,float16,0,0.011722666521867117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,24,128,1,float16,float16,0,0.011909333368142446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,1,128,1,float16,float16,0,0.011514666179815928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,2,128,1,float16,float16,0,0.01173866664369901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,4,128,1,float16,float16,0,0.011898666620254517
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,8,128,1,float16,float16,0,0.011674666156371435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,1,128,1,float16,float16,0,0.11778133114178975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,2,128,1,float16,float16,0,0.1243839959303538
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,24,128,1,float16,float16,0,0.1469386617342631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,24,1,128,1,float16,float16,0,0.22592532634735107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,24,2,128,1,float16,float16,0,0.238154669602712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,24,8,128,1,float16,float16,0,0.2644853393236796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,24,4,128,1,float16,float16,0,0.26268800099690753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,24,128,1,float16,float16,0,0.07976533472537994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,8,128,1,float16,float16,0,0.1388053297996521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,4,128,1,float16,float16,0,0.13749866684277853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,1,128,1,float16,float16,0,0.0598826656738917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,2,128,1,float16,float16,0,0.06151466568311056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,4,128,1,float16,float16,0,0.06579733391602834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,24,128,1,float16,float16,0,0.0400693342089653
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,8,128,1,float16,float16,0,0.06778666873772939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,4,128,1,float16,float16,0,0.038106667498747505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,1,128,1,float16,float16,0,0.034389334420363106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,8,128,1,float16,float16,0,0.038165333370367684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,2,128,1,float16,float16,0,0.036015999813874565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,24,128,1,float16,float16,0,0.0240639994541804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,4,128,1,float16,float16,0,0.023455999791622162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,8,128,1,float16,float16,0,0.023957334458827972
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,1,128,1,float16,float16,0,0.021498667697111767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,24,128,1,float16,float16,0,0.01565333331624667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,2,128,1,float16,float16,0,0.02170666555563609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,1,128,1,float16,float16,0,0.014650666465361914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,4,128,1,float16,float16,0,0.015141333142916361
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,8,128,1,float16,float16,0,0.015034666905800501
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,1,128,1,float16,float16,0,0.01191466674208641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,2,128,1,float16,float16,0,0.011930666863918304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,2,128,1,float16,float16,0,0.014682666709025701
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,4,128,1,float16,float16,0,0.012058666596810022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,24,128,1,float16,float16,0,0.012362666428089142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,1,128,1,float16,float16,0,0.011525332927703857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,24,128,1,float16,float16,0,0.012165332833925882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,2,128,1,float16,float16,0,0.011695999652147293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,4,128,1,float16,float16,0,0.011765333513418833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,8,128,1,float16,float16,0,0.011749333391586939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,24,128,1,float16,float16,0,0.011946666985750198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,2,128,1,float16,float16,0,0.011381333072980246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,8,128,1,float16,float16,0,0.011754666765530905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,4,128,1,float16,float16,0,0.011589333415031433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,1,128,1,float16,float16,0,0.011557333171367645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,0,0.016442666451136272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,0,0.027797333896160126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,24,128,1,float16,float16,0,0.1009386678536733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,0,0.02094399929046631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,0,0.038149334490299225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,24,128,1,float16,float16,0,0.055871998270352684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,0,0.05554133156935374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,0,0.03481066723664602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,0,0.01333333303531011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,0,0.08690667152404785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,0,0.05099200208981832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,0,0.01321600005030632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,0,0.019685332973798115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,24,128,1,float16,float16,0,0.033520000676314034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,0,0.022730665902296703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,0,0.033386667569478355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,0,0.027237333357334137
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,0,0.022469334304332733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,0,0.02683199942111969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,0,0.013162666310866674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,24,128,1,float16,float16,0,0.020954666038354237
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,0,0.019674666225910187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,0,0.01923199991385142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,0,0.012703999876976013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,24,128,1,float16,float16,0,0.01402666668097178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,0,0.02221333235502243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,0,0.018757333358128864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,0,0.018911999960740406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,0,0.019248000035683315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,0,0.018320000420014065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,0,0.012565333396196365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,0,0.018863999595244724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,24,128,1,float16,float16,0,0.01139733319481214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,24,128,1,float16,float16,0,0.011226666470368704
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,0,0.018645333747069042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,0,0.012549333274364471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,0,0.01836266616980235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,24,128,1,float16,float16,0,0.011247999966144562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,0,0.018640000373125076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,0,0.012304000556468964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,0,0.01246400053302447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,0,0.012479999413092932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,0,0.018288000176350277
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,0,0.012538666526476542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,1,128,1,float16,float16,0,24.29132843017578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,2,128,1,float16,float16,0,24.724522908528645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,16,128,1,float16,float16,0,25.613665262858074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,4,128,1,float16,float16,0,25.41070810953776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,1,128,1,float16,float16,0,12.480042775472006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,16,128,1,float16,float16,0,12.764687856038412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,2,128,1,float16,float16,0,12.521733601888021
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,16,128,1,float16,float16,0,6.4643198649088545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,16,1,128,1,float16,float16,0,49.153361002604164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,16,2,128,1,float16,float16,0,49.11561075846354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,8,128,1,float16,float16,0,25.01958974202474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,16,4,128,1,float16,float16,0,50.51806640625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,16,8,128,1,float16,float16,0,50.2421620686849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,4,128,1,float16,float16,0,12.791685740152994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,8,128,1,float16,float16,0,12.651055653889975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,1,128,1,float16,float16,0,6.266805648803711
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,2,128,1,float16,float16,0,6.2357330322265625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,4,128,1,float16,float16,0,6.28221321105957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,8,128,1,float16,float16,0,6.482074737548828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,16,128,1,float16,float16,0,14.596922556559244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,1,128,1,float16,float16,0,13.78439458211263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,2,128,1,float16,float16,0,14.263621012369791
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,16,1,128,1,float16,float16,0,28.04351552327474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,16,128,1,float16,float16,0,7.275285085042317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,1,128,1,float16,float16,0,6.986597061157227
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,4,128,1,float16,float16,0,14.37509282430013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,16,4,128,1,float16,float16,0,29.051055908203125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,16,2,128,1,float16,float16,0,28.743451436360676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,16,8,128,1,float16,float16,0,29.46404774983724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,8,128,1,float16,float16,0,14.598197937011719
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,2,128,1,float16,float16,0,7.0568586985270185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,1,128,1,float16,float16,0,3.5371360778808594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,2,128,1,float16,float16,0,3.595370610555013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,16,128,1,float16,float16,0,3.643797238667806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,4,128,1,float16,float16,0,7.107184092203776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,4,128,1,float16,float16,0,3.672368049621582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,8,128,1,float16,float16,0,7.282623926798503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,8,128,1,float16,float16,0,3.526421229044596
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,1,128,1,float16,float16,0,9.905402501424154
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,16,128,1,float16,float16,0,10.170010884602865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,2,128,1,float16,float16,0,10.039040247599283
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,4,128,1,float16,float16,0,10.176954905192057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,16,128,1,float16,float16,0,5.043509483337402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,1,128,1,float16,float16,0,4.91702938079834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,16,1,128,1,float16,float16,0,19.57084782918294
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,16,2,128,1,float16,float16,0,19.68132781982422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,16,4,128,1,float16,float16,0,20.23853302001953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,16,8,128,1,float16,float16,0,20.09601593017578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,8,128,1,float16,float16,0,10.22709846496582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,2,128,1,float16,float16,0,4.934842745463054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,4,128,1,float16,float16,0,5.045968055725098
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,16,128,1,float16,float16,0,2.658517360687256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,1,128,1,float16,float16,0,2.5010132789611816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,8,128,1,float16,float16,0,5.0570878982543945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,2,128,1,float16,float16,0,2.551583925882975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,4,128,1,float16,float16,0,2.658677260080973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,8,128,1,float16,float16,0,2.6564265886942544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,1,128,1,float16,float16,0,12.843509674072266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,16,128,1,float16,float16,0,13.496341705322266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,2,128,1,float16,float16,0,12.766661326090494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,4,128,1,float16,float16,0,13.358133951822916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,1,128,1,float16,float16,0,6.213461558024089
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,16,128,1,float16,float16,0,6.682853062947591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,2,128,1,float16,float16,0,6.386325200398763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,16,1,128,1,float16,float16,0,25.211835225423176
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,16,2,128,1,float16,float16,0,25.321617126464844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,16,128,1,float16,float16,0,3.3705708185831704
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,16,4,128,1,float16,float16,0,26.445653279622395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,16,8,128,1,float16,float16,0,26.52123260498047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,1,128,1,float16,float16,0,3.180901209513346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,8,128,1,float16,float16,0,13.444997151692709
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,4,128,1,float16,float16,0,6.449941635131836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,2,128,1,float16,float16,0,3.209930737813314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,16,128,1,float16,float16,0,1.8020480473836262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,8,128,1,float16,float16,0,6.657503763834636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,1,128,1,float16,float16,0,1.6252373059590657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,4,128,1,float16,float16,0,3.3244479497273765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,2,128,1,float16,float16,0,1.6922240257263184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,8,128,1,float16,float16,0,3.2939093907674155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,4,128,1,float16,float16,0,1.7331040700276692
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,8,128,1,float16,float16,0,1.7041066487630208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,1,128,1,float16,float16,0,7.217706680297852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,2,128,1,float16,float16,0,7.281904220581055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,16,128,1,float16,float16,0,7.838789621988933
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,4,128,1,float16,float16,0,7.5023148854573565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,1,128,1,float16,float16,0,3.5729173024495444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,16,128,1,float16,float16,0,3.8528159459431968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,2,128,1,float16,float16,0,3.640037218729655
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,16,1,128,1,float16,float16,0,14.468021392822266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,16,2,128,1,float16,float16,0,14.773818969726562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,16,4,128,1,float16,float16,0,15.29433568318685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,16,128,1,float16,float16,0,1.9458400408426921
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,16,8,128,1,float16,float16,0,15.560240427652994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,8,128,1,float16,float16,0,7.652890523274739
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,1,128,1,float16,float16,0,1.8314132690429688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,2,128,1,float16,float16,0,1.8809599876403809
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,4,128,1,float16,float16,0,3.7848374048868814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,4,128,1,float16,float16,0,1.9353173573811848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,16,128,1,float16,float16,0,1.07260266939799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,8,128,1,float16,float16,0,3.7092374165852866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,1,128,1,float16,float16,0,1.0210453669230144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,2,128,1,float16,float16,0,0.9937493006388346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,8,128,1,float16,float16,0,1.8942666053771973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,4,128,1,float16,float16,0,1.0217866897583008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,8,128,1,float16,float16,0,1.0479893684387207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,1,128,1,float16,float16,0,6.4828535715738935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,16,128,1,float16,float16,0,7.3652801513671875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,2,128,1,float16,float16,0,6.815914789835612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,4,128,1,float16,float16,0,7.215285619099935
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,16,128,1,float16,float16,0,3.6517438888549805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,1,128,1,float16,float16,0,3.2741174697875977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,2,128,1,float16,float16,0,3.3342294692993164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,16,1,128,1,float16,float16,0,13.260826110839844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,16,2,128,1,float16,float16,0,13.847562154134115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,16,4,128,1,float16,float16,0,14.268831888834635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,16,8,128,1,float16,float16,0,14.3745969136556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,8,128,1,float16,float16,0,7.063488006591797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,1,128,1,float16,float16,0,1.669424057006836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,16,128,1,float16,float16,0,1.849023977915446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,4,128,1,float16,float16,0,3.4927358627319336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,8,128,1,float16,float16,0,3.574085235595703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,2,128,1,float16,float16,0,1.6855519612630208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,4,128,1,float16,float16,0,1.8424213727315266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,16,128,1,float16,float16,0,0.9617439905802408
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,1,128,1,float16,float16,0,0.8483093579610189
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,8,128,1,float16,float16,0,1.81441068649292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,16,128,1,float16,float16,0,0.5297706524531046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,2,128,1,float16,float16,0,0.8921759923299154
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,1,128,1,float16,float16,0,0.47164801756540936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,4,128,1,float16,float16,0,0.9317440191904703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,8,128,1,float16,float16,0,0.9452319939931234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,2,128,1,float16,float16,0,0.4744906822840373
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,4,128,1,float16,float16,0,0.5146079858144125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,8,128,1,float16,float16,0,0.5106133222579956
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,1,128,1,float16,float16,0,3.659125328063965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,2,128,1,float16,float16,0,3.8140319188435874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,16,128,1,float16,float16,0,4.279024124145508
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,16,128,1,float16,float16,0,2.1653332710266113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,16,2,128,1,float16,float16,0,7.967983881632487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,16,4,128,1,float16,float16,0,8.569061279296875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,16,8,128,1,float16,float16,0,8.465765635172525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,16,1,128,1,float16,float16,0,7.7531890869140625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,4,128,1,float16,float16,0,4.084719975789388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,8,128,1,float16,float16,0,4.179967880249023
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,2,128,1,float16,float16,0,1.912501335144043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,16,128,1,float16,float16,0,1.1496533552805583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,4,128,1,float16,float16,0,2.136298656463623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,1,128,1,float16,float16,0,0.9754026730855306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,2,128,1,float16,float16,0,1.022602637608846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,4,128,1,float16,float16,0,1.0785706837972004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,8,128,1,float16,float16,0,2.135258674621582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,16,128,1,float16,float16,0,0.5923306544621786
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,1,128,1,float16,float16,0,0.5075200001398722
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,8,128,1,float16,float16,0,1.1022666295369465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,1,128,1,float16,float16,0,1.7933813730875652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,16,128,1,float16,float16,0,0.33742932478586835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,2,128,1,float16,float16,0,0.29895999034245807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,4,128,1,float16,float16,0,0.5743093490600586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,8,128,1,float16,float16,0,0.5749599933624268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,2,128,1,float16,float16,0,0.5369439919789633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,4,128,1,float16,float16,0,0.3192480007807414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,1,128,1,float16,float16,0,0.28857600688934326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,8,128,1,float16,float16,0,0.31304534276326496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,1,128,1,float16,float16,0,3.3945706685384116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,16,128,1,float16,float16,0,4.160239855448405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,4,128,1,float16,float16,0,3.982821464538574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,2,128,1,float16,float16,0,3.590693473815918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,16,128,1,float16,float16,0,2.167952060699463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,16,1,128,1,float16,float16,0,7.297605514526367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,1,128,1,float16,float16,0,1.7104907035827637
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,16,4,128,1,float16,float16,0,8.211909612019857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,16,8,128,1,float16,float16,0,8.387503941853842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,16,2,128,1,float16,float16,0,7.593749364217122
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,2,128,1,float16,float16,0,1.8459466298421223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,8,128,1,float16,float16,0,4.069445292154948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,1,128,1,float16,float16,0,0.9057226975758871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,16,128,1,float16,float16,0,1.0780320167541504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,2,128,1,float16,float16,0,0.9984320004781088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,4,128,1,float16,float16,0,2.0215147336324057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,16,128,1,float16,float16,0,0.5703786611557007
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,8,128,1,float16,float16,0,2.14410130182902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,1,128,1,float16,float16,0,0.46747732162475586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,2,128,1,float16,float16,0,0.4894346793492635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,8,128,1,float16,float16,0,0.5468746821085612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,16,128,1,float16,float16,0,0.30772799253463745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,1,128,1,float16,float16,0,0.25960532824198407
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,2,128,1,float16,float16,0,0.2701706687609355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,8,128,1,float16,float16,0,1.0450987021128337
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,4,128,1,float16,float16,0,0.5475466648737589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,16,128,1,float16,float16,0,0.17814934253692627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,8,128,1,float16,float16,0,0.2993653416633606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,4,128,1,float16,float16,0,0.16141866644223532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,2,128,1,float16,float16,0,0.14994666973749796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,8,128,1,float16,float16,0,0.17072532574335733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,1,128,1,float16,float16,0,0.14789332946141562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,4,128,1,float16,float16,0,0.29074666897455853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,4,128,1,float16,float16,0,1.0354186693827312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,1,128,1,float16,float16,0,2.0475786526997886
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,16,128,1,float16,float16,0,2.6284799575805664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,4,128,1,float16,float16,0,2.486623922983805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,16,128,1,float16,float16,0,1.3540372848510742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,16,1,128,1,float16,float16,0,4.068149248758952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,2,128,1,float16,float16,0,2.139125347137451
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,16,4,128,1,float16,float16,0,4.930826822916667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,1,128,1,float16,float16,0,1.0260480244954426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,16,8,128,1,float16,float16,0,5.009087880452474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,16,2,128,1,float16,float16,0,4.343423843383789
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,8,128,1,float16,float16,0,2.5629067420959473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,1,128,1,float16,float16,0,0.5264106591542562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,16,128,1,float16,float16,0,0.6913066705067953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,2,128,1,float16,float16,0,0.582266648610433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,2,128,1,float16,float16,0,1.108026663462321
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,8,128,1,float16,float16,0,0.6636159817377726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,16,128,1,float16,float16,0,0.35951467355092365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,4,128,1,float16,float16,0,1.29694930712382
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,1,128,1,float16,float16,0,0.2881973385810852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,8,128,1,float16,float16,0,1.259434700012207
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,4,128,1,float16,float16,0,0.34935466448465985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,2,128,1,float16,float16,0,0.3052746653556824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,16,128,1,float16,float16,0,0.20070399840672812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,4,128,1,float16,float16,0,0.6623786687850952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,8,128,1,float16,float16,0,0.3472213347752889
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,1,128,1,float16,float16,0,0.16081600387891135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,2,128,1,float16,float16,0,0.16952532529830933
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,4,128,1,float16,float16,0,0.19156267245610556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,16,128,1,float16,float16,0,0.11981333295504253
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,8,128,1,float16,float16,0,0.19368000825246176
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,2,128,1,float16,float16,0,0.1018506685892741
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,1,128,1,float16,float16,0,0.099781334400177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,4,128,1,float16,float16,0,0.10855467120806377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,8,128,1,float16,float16,0,0.11172800262769063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,16,128,1,float16,float16,0,2.7880214055379233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,1,128,1,float16,float16,0,1.995408058166504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,16,1,128,1,float16,float16,0,3.9610134760538735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,2,128,1,float16,float16,0,2.163813273111979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,16,2,128,1,float16,float16,0,4.382320086161296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,8,128,1,float16,float16,0,2.6549280484517417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,16,8,128,1,float16,float16,0,5.278853416442871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,4,128,1,float16,float16,0,2.6234453519185386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,1,128,1,float16,float16,0,1.0058506329854329
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,8,128,1,float16,float16,0,1.3501440684000652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,16,128,1,float16,float16,0,1.4237386385599773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,4,128,1,float16,float16,0,1.317903995513916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,2,128,1,float16,float16,0,1.1321813265482585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,16,4,128,1,float16,float16,0,5.237792015075684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,1,128,1,float16,float16,0,0.5172853469848633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,2,128,1,float16,float16,0,0.5704266627629598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,16,128,1,float16,float16,0,0.3772746721903483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,4,128,1,float16,float16,0,0.6757973035176595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,1,128,1,float16,float16,0,0.2725546757380168
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,4,128,1,float16,float16,0,0.35098667939503986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,16,128,1,float16,float16,0,0.7268533706665039
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,16,128,1,float16,float16,0,0.2039573391278585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,8,128,1,float16,float16,0,0.6787412961324056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,8,128,1,float16,float16,0,0.3587413231531779
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,1,128,1,float16,float16,0,0.1537920037905375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,2,128,1,float16,float16,0,0.16390400131543478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,4,128,1,float16,float16,0,0.1891253391901652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,1,128,1,float16,float16,0,0.086325337489446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,16,128,1,float16,float16,0,0.11874666810035706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,2,128,1,float16,float16,0,0.09032000104586284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,8,128,1,float16,float16,0,0.19684267044067383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,4,128,1,float16,float16,0,0.10187733173370361
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,8,128,1,float16,float16,0,0.11158399780591328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,8,128,1,float16,float16,0,0.06133866806825002
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,4,128,1,float16,float16,0,0.06102400024731954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,1,128,1,float16,float16,0,0.055200000603993736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,2,128,1,float16,float16,0,0.05699733396371206
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,16,128,1,float16,float16,0,0.06313600142796834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,2,128,1,float16,float16,0,0.30264000097910565
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,1,128,1,float16,float16,0,1.3049866358439128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,16,128,1,float16,float16,0,2.0900959968566895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,2,128,1,float16,float16,0,1.4968640009562175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,16,128,1,float16,float16,0,1.05567463239034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,16,1,128,1,float16,float16,0,2.577125390370687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,4,128,1,float16,float16,0,1.9278346697489421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,1,128,1,float16,float16,0,0.6572959820429484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,4,128,1,float16,float16,0,0.9686880111694336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,8,128,1,float16,float16,0,0.9988799889882406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,2,128,1,float16,float16,0,0.7636640071868896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,16,2,128,1,float16,float16,0,2.9375734329223633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,16,128,1,float16,float16,0,0.5417333443959554
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,1,128,1,float16,float16,0,0.3369226853052775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,4,128,1,float16,float16,0,0.4934186538060506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,2,128,1,float16,float16,0,0.3953813314437866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,16,128,1,float16,float16,0,0.2825813293457031
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,1,128,1,float16,float16,0,0.17970667282740274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,8,128,1,float16,float16,0,0.5039039850234985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,4,128,1,float16,float16,0,0.25893332560857135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,2,128,1,float16,float16,0,0.2058560053507487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,16,128,1,float16,float16,0,0.15321066975593567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,8,128,1,float16,float16,0,0.2632960081100464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,8,128,1,float16,float16,0,1.9568640391031902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,4,128,1,float16,float16,0,0.13829867045084634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,2,128,1,float16,float16,0,0.11459733049074809
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,8,128,1,float16,float16,0,0.14289066195487976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,16,128,1,float16,float16,0,0.08668800195058186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,1,128,1,float16,float16,0,0.10233599940935771
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,1,128,1,float16,float16,0,0.05714133381843567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,4,128,1,float16,float16,0,0.0713973343372345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,16,128,1,float16,float16,0,0.048538664976755776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,8,128,1,float16,float16,0,0.07991999884446462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,2,128,1,float16,float16,0,0.04012800008058548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,1,128,1,float16,float16,0,0.03868266691764196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,4,128,1,float16,float16,0,0.04475200176239014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,16,128,1,float16,float16,0,0.03089066594839096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,8,128,1,float16,float16,0,0.04497066636880239
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,1,128,1,float16,float16,0,0.02720533311367035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,4,128,1,float16,float16,0,0.03057066599527995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,2,128,1,float16,float16,0,0.02874133239189784
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,8,128,1,float16,float16,0,0.030634666482607525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,16,4,128,1,float16,float16,0,3.7986294428507485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,2,128,1,float16,float16,0,0.06088533500830332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,16,8,128,1,float16,float16,0,3.891397476196289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,1,128,1,float16,float16,0,0.4854559898376465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,16,2,128,1,float16,float16,0,1.1402239799499512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,4,128,1,float16,float16,0,0.7925013701121012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,2,128,1,float16,float16,0,0.5907626549402872
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,16,128,1,float16,float16,0,0.881055990854899
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,16,1,128,1,float16,float16,0,0.9625493685404459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,16,4,128,1,float16,float16,0,1.583082675933838
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,16,128,1,float16,float16,0,0.44914666811625165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,2,128,1,float16,float16,0,0.2980159918467204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,1,128,1,float16,float16,0,0.24290666977564493
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,8,128,1,float16,float16,0,0.8186346689860026
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,16,8,128,1,float16,float16,0,1.6022666295369465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,4,128,1,float16,float16,0,0.2076746622721354
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,2,128,1,float16,float16,0,0.15685866276423135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,1,128,1,float16,float16,0,0.07468266785144806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,2,128,1,float16,float16,0,0.08729066451390584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,8,128,1,float16,float16,0,0.40985600153605145
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,4,128,1,float16,float16,0,0.11129066348075867
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,8,128,1,float16,float16,0,0.1165013313293457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,1,128,1,float16,float16,0,0.1307146648565928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,1,128,1,float16,float16,0,0.04165333261092504
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,8,128,1,float16,float16,0,0.06505600114663442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,2,128,1,float16,float16,0,0.04585599899291992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,4,128,1,float16,float16,0,0.055904000997543335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,4,128,1,float16,float16,0,0.4001493453979492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,16,128,1,float16,float16,0,0.23577600717544556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,16,128,1,float16,float16,0,0.035274667044480644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,1,128,1,float16,float16,0,0.02693866689999898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,2,128,1,float16,float16,0,0.02849599967400233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,8,128,1,float16,float16,0,0.21645333369572958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,1,128,1,float16,float16,0,0.021162666380405426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,16,128,1,float16,float16,0,0.02497600018978119
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,2,128,1,float16,float16,0,0.022416000564893086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,16,128,1,float16,float16,0,0.07148799796899159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,8,128,1,float16,float16,0,0.024714666108290356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,1,128,1,float16,float16,0,0.01575999955336253
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,16,128,1,float16,float16,0,0.01800000046690305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,4,128,1,float16,float16,0,0.017914666483799618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,8,128,1,float16,float16,0,0.01786133274435997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,2,128,1,float16,float16,0,0.016106666376193363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,4,128,1,float16,float16,0,0.03238933285077413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,4,128,1,float16,float16,0,0.024608001112937927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,8,128,1,float16,float16,0,0.03306666761636734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,1,128,1,float16,float16,0,0.19201600551605225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,2,128,1,float16,float16,0,0.24895467360814413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,16,128,1,float16,float16,0,0.12637866536776224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,16,1,128,1,float16,float16,0,0.4395466645558675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,16,2,128,1,float16,float16,0,0.5389333168665568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,4,128,1,float16,float16,0,0.35623466968536377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,16,128,1,float16,float16,0,0.216538667678833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,16,128,1,float16,float16,0,0.4230506817499797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,8,128,1,float16,float16,0,0.37098666032155353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,16,4,128,1,float16,float16,0,0.7395733197530111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,4,128,1,float16,float16,0,0.1825653314590454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,8,128,1,float16,float16,0,0.19182932376861572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,16,8,128,1,float16,float16,0,0.7627413272857666
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,1,128,1,float16,float16,0,0.10392533739407857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,4,128,1,float16,float16,0,0.09794666369756062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,2,128,1,float16,float16,0,0.13060800234476724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,8,128,1,float16,float16,0,0.10364266236623128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,1,128,1,float16,float16,0,0.06044800082842509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,16,128,1,float16,float16,0,0.11601600050926208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,1,128,1,float16,float16,0,0.03294933338960012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,2,128,1,float16,float16,0,0.03636800001064936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,2,128,1,float16,float16,0,0.024485332270463307
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,16,128,1,float16,float16,0,0.03211733450492223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,4,128,1,float16,float16,0,0.04828799764315287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,8,128,1,float16,float16,0,0.05715199808279673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,4,128,1,float16,float16,0,0.02808533360560735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,8,128,1,float16,float16,0,0.028607999285062153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,1,128,1,float16,float16,0,0.022783999641736347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,16,128,1,float16,float16,0,0.019989332805077236
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,2,128,1,float16,float16,0,0.07207466661930084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,1,128,1,float16,float16,0,0.01588800052801768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,2,128,1,float16,float16,0,0.01757866640885671
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,4,128,1,float16,float16,0,0.019498666127522785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,8,128,1,float16,float16,0,0.019706666469573975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,16,128,1,float16,float16,0,0.014581333845853806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,1,128,1,float16,float16,0,0.012522666404644648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,4,128,1,float16,float16,0,0.01440000037352244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,8,128,1,float16,float16,0,0.014629332969586054
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,2,128,1,float16,float16,0,0.012639999389648438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,4,128,1,float16,float16,0,0.01267733300725619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,1,128,1,float16,float16,0,0.012437333663304647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,2,128,1,float16,float16,0,0.0122079998254776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,8,128,1,float16,float16,0,0.012645332763592402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,16,128,1,float16,float16,0,0.013061333447694778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,16,128,1,float16,float16,0,0.06584533552328746
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,1,128,1,float16,float16,0,0.10424533486366272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,4,128,1,float16,float16,0,0.18239466349283853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,2,128,1,float16,float16,0,0.13106133540471396
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,16,128,1,float16,float16,0,0.21310933430989584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,16,128,1,float16,float16,0,0.11618666847546895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,16,1,128,1,float16,float16,0,0.18971200784047446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,1,128,1,float16,float16,0,0.0601440022389094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,16,2,128,1,float16,float16,0,0.24306132396062216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,8,128,1,float16,float16,0,0.18896534045537314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,16,4,128,1,float16,float16,0,0.3522293170293172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,2,128,1,float16,float16,0,0.07267199953397115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,16,128,1,float16,float16,0,0.06465066472689311
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,16,8,128,1,float16,float16,0,0.36454931894938153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,4,128,1,float16,float16,0,0.097653329372406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,4,128,1,float16,float16,0,0.04574400186538696
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,8,128,1,float16,float16,0,0.10359999537467957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,8,128,1,float16,float16,0,0.05841066439946493
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,1,128,1,float16,float16,0,0.03137599925200144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,1,128,1,float16,float16,0,0.02202133337656657
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,2,128,1,float16,float16,0,0.023957334458827972
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,2,128,1,float16,float16,0,0.035061334570248924
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,16,128,1,float16,float16,0,0.019519999623298645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,8,128,1,float16,float16,0,0.02842666705449422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,4,128,1,float16,float16,0,0.02754133443037669
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,1,128,1,float16,float16,0,0.015781333049138386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,2,128,1,float16,float16,0,0.017456000049908955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,16,128,1,float16,float16,0,0.030693332354227703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,4,128,1,float16,float16,0,0.019466667125622433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,8,128,1,float16,float16,0,0.01940800001223882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,1,128,1,float16,float16,0,0.012469333906968435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,16,128,1,float16,float16,0,0.014767999450365702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,4,128,1,float16,float16,0,0.01444799949725469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,2,128,1,float16,float16,0,0.012671999633312225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,16,128,1,float16,float16,0,0.012885333349307379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,1,128,1,float16,float16,0,0.01191466674208641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,8,128,1,float16,float16,0,0.014607999473810196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,2,128,1,float16,float16,0,0.012165332833925882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,8,128,1,float16,float16,0,0.01268799975514412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,4,128,1,float16,float16,0,0.012602667013804117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,16,128,1,float16,float16,0,0.012128000458081564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,2,128,1,float16,float16,0,0.01179733375708262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,8,128,1,float16,float16,0,0.012138667205969492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,4,128,1,float16,float16,0,0.011989332735538483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,1,128,1,float16,float16,0,0.09102933605511983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,16,128,1,float16,float16,0,0.13934399684270224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,2,128,1,float16,float16,0,0.10382399956385295
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,16,1,128,1,float16,float16,0,0.16762133439381918
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,16,8,128,1,float16,float16,0,0.2499306599299113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,16,2,128,1,float16,float16,0,0.19344000021616617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,16,128,1,float16,float16,0,0.0773173322280248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,4,128,1,float16,float16,0,0.12780800461769104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,16,4,128,1,float16,float16,0,0.2437653342882792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,8,128,1,float16,float16,0,0.1306933363278707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,1,128,1,float16,float16,0,0.04525866607824961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,16,128,1,float16,float16,0,0.03968533376852671
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,4,128,1,float16,float16,0,0.033258666594823204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,8,128,1,float16,float16,0,0.07130133112271626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,2,128,1,float16,float16,0,0.02942933390537898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,8,128,1,float16,float16,0,0.03382399926582972
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,4,128,1,float16,float16,0,0.05961599946022034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,16,128,1,float16,float16,0,0.023365333676338196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,1,128,1,float16,float16,0,0.027727998793125153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,4,128,1,float16,float16,0,0.023232000569502514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,2,128,1,float16,float16,0,0.020874666670958202
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,2,128,1,float16,float16,0,0.049423997600873314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,16,128,1,float16,float16,0,0.016666666915019352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,1,128,1,float16,float16,0,0.01952533299724261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,8,128,1,float16,float16,0,0.023408000667889912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,1,128,1,float16,float16,0,0.014885333677132925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,2,128,1,float16,float16,0,0.014965333044528961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,4,128,1,float16,float16,0,0.01648533344268799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,8,128,1,float16,float16,0,0.01659199967980385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,16,128,1,float16,float16,0,0.012650666137536367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,2,128,1,float16,float16,0,0.012042666474978128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,1,128,1,float16,float16,0,0.011893333246310553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,4,128,1,float16,float16,0,0.012378666549921036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,8,128,1,float16,float16,0,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,16,128,1,float16,float16,0,0.01190399999419848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,1,128,1,float16,float16,0,0.011535999675591787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,4,128,1,float16,float16,0,0.011850666254758835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,8,128,1,float16,float16,0,0.011957333733638128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,16,128,1,float16,float16,0,0.011941333611806234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,4,128,1,float16,float16,0,0.011567999919255575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,2,128,1,float16,float16,0,0.011610666910807291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,2,128,1,float16,float16,0,0.011589333415031433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,8,128,1,float16,float16,0,0.011658667276302973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,2,128,1,float16,float16,0,0.08063999811808269
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,4,128,1,float16,float16,0,0.09379733602205913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,1,128,1,float16,float16,0,0.0772159993648529
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,16,128,1,float16,float16,0,0.10825600226720174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,16,128,1,float16,float16,0,0.05384533107280731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,16,1,128,1,float16,float16,0,0.15565866231918335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,16,2,128,1,float16,float16,0,0.16891199350357056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,1,128,1,float16,float16,0,0.04275733232498169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,16,8,128,1,float16,float16,0,0.19640000661214194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,2,128,1,float16,float16,0,0.044495999813079834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,16,4,128,1,float16,float16,0,0.1930933396021525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,8,128,1,float16,float16,0,0.10193066795667012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,4,128,1,float16,float16,0,0.048911998669306435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,8,128,1,float16,float16,0,0.049173335234324135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,1,128,1,float16,float16,0,0.025920001169045765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,4,128,1,float16,float16,0,0.029525332152843475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,16,128,1,float16,float16,0,0.02997333308060964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,16,128,1,float16,float16,0,0.021402666966120403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,2,128,1,float16,float16,0,0.02737066646416982
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,2,128,1,float16,float16,0,0.019167999426523846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,8,128,1,float16,float16,0,0.02994133283694585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,1,128,1,float16,float16,0,0.019023999571800232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,8,128,1,float16,float16,0,0.021082667013009388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,4,128,1,float16,float16,0,0.02093333254257838
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,1,128,1,float16,float16,0,0.014458666245142618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,2,128,1,float16,float16,0,0.014442666123310724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,16,128,1,float16,float16,0,0.015119999647140503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,8,128,1,float16,float16,0,0.01498666654030482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,4,128,1,float16,float16,0,0.015008000036080679
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,16,128,1,float16,float16,0,0.011941333611806234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,2,128,1,float16,float16,0,0.011616000284751257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,4,128,1,float16,float16,0,0.012085333466529846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,1,128,1,float16,float16,0,0.011509332805871964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,8,128,1,float16,float16,0,0.012053333222866058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,16,128,1,float16,float16,0,0.011930666863918304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,1,128,1,float16,float16,0,0.011440000186363855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,4,128,1,float16,float16,0,0.011711999773979187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,2,128,1,float16,float16,0,0.011706666400035223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,8,128,1,float16,float16,0,0.011600000162919363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,2,128,1,float16,float16,0,0.011407999942700068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,4,128,1,float16,float16,0,0.011765333513418833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,1,128,1,float16,float16,0,0.011584000041087469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,8,128,1,float16,float16,0,0.011663999408483505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,16,128,1,float16,float16,0,0.01192533348997434
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,0,0.0161013330022494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,0,0.027322667340437572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,16,128,1,float16,float16,0,0.07085333267847697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,0,0.020560000091791153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,0,0.03603733330965042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,0,0.05387733379999796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,16,128,1,float16,float16,0,0.040549332896868386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,0,0.08432533343633015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,0,0.049546668926874794
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,0,0.0341386670867602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,0,0.01322666679819425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,16,128,1,float16,float16,0,0.02477866659561793
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,0,0.026965332527955372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,0,0.02235200007756551
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,0,0.03332799921433131
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,0,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,0,0.019744000087181728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,0,0.022005334496498108
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,0,0.026837334036827087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,16,128,1,float16,float16,0,0.018144000321626663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,0,0.012869333227475485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,0,0.019578666736682255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,0,0.019653332730134327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,0,0.02195200075705846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,16,128,1,float16,float16,0,0.014101333916187286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,0,0.012608000387748083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,0,0.01848000039656957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,0,0.018565333137909572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,0,0.018901333212852478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,16,128,1,float16,float16,0,0.011087999989589056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,0,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,0,0.018245333184798557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,0,0.018629333625237148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,0,0.018464000274737675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,16,128,1,float16,float16,0,0.010981333752473196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,0,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,0,0.01817600056529045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,0,0.018570666511853535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,0,0.018144000321626663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,16,128,1,float16,float16,0,0.011007999380429586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,0,0.012138667205969492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,0,0.012282667060693106
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,0,0.012554666648308435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,0,0.0124746672809124
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,12,12,128,1,float16,float16,0,9.627632141113281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,12,1,128,1,float16,float16,0,18.350229899088543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,12,2,128,1,float16,float16,0,18.575552622477215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,12,4,128,1,float16,float16,0,18.86522165934245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,12,12,128,1,float16,float16,0,19.563610076904297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,12,12,128,1,float16,float16,0,4.901151974995931
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,12,1,128,1,float16,float16,0,8.802661259969076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,12,1,128,1,float16,float16,0,4.760149319966634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,12,2,128,1,float16,float16,0,9.171914418538412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,12,4,128,1,float16,float16,0,9.522591908772787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,12,2,128,1,float16,float16,0,4.685717264811198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,12,4,128,1,float16,float16,0,4.789888064066569
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,12,1,128,1,float16,float16,0,37.051536560058594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,12,2,128,1,float16,float16,0,37.258201599121094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,12,4,128,1,float16,float16,0,38.24516805013021
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,12,1,128,1,float16,float16,0,10.4095090230306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,12,12,128,1,float16,float16,0,10.971509297688803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,12,12,128,1,float16,float16,0,5.59986686706543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,12,1,128,1,float16,float16,0,21.26430384318034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,12,1,128,1,float16,float16,0,5.2255252202351885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,12,2,128,1,float16,float16,0,5.258048057556152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,12,2,128,1,float16,float16,0,10.71011225382487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,12,4,128,1,float16,float16,0,10.899856567382812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,12,2,128,1,float16,float16,0,21.2858403523763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,12,12,128,1,float16,float16,0,2.927760124206543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,12,4,128,1,float16,float16,0,5.54251225789388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,12,1,128,1,float16,float16,0,2.71833070119222
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,12,4,128,1,float16,float16,0,21.846463521321613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,12,2,128,1,float16,float16,0,2.7082932790120444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,12,4,128,1,float16,float16,0,2.8542505900065103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,12,12,128,1,float16,float16,0,3.9182399113972983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,12,1,128,1,float16,float16,0,7.304831822713216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,12,2,128,1,float16,float16,0,7.38536008199056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,12,12,128,1,float16,float16,0,7.795066833496094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,12,4,128,1,float16,float16,0,7.6286881764729815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,12,1,128,1,float16,float16,0,3.695018768310547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,12,12,128,1,float16,float16,0,2.063151995340983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,12,1,128,1,float16,float16,0,1.9507733980814617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,12,2,128,1,float16,float16,0,3.8619254430135093
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,12,1,128,1,float16,float16,0,14.759941101074219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,12,4,128,1,float16,float16,0,3.9251893361409507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,12,2,128,1,float16,float16,0,1.9509387016296387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,12,2,128,1,float16,float16,0,14.963333129882812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,12,4,128,1,float16,float16,0,15.624725341796875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,12,4,128,1,float16,float16,0,1.981210708618164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,12,1,128,1,float16,float16,0,9.542624155680338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,12,12,128,1,float16,float16,0,10.101450602213541
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,12,2,128,1,float16,float16,0,9.869471867879232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,12,4,128,1,float16,float16,0,10.112405141194662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,12,12,128,1,float16,float16,0,2.5988853772481284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,12,1,128,1,float16,float16,0,4.689733187357585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,12,2,128,1,float16,float16,0,4.726586659749349
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,12,12,128,1,float16,float16,0,4.915989240010579
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,12,4,128,1,float16,float16,0,5.048933347066243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,12,1,128,1,float16,float16,0,18.83894983927409
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,12,2,128,1,float16,float16,0,19.473103841145832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,12,1,128,1,float16,float16,0,2.3872960408528647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,12,12,128,1,float16,float16,0,1.374911944071452
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,12,2,128,1,float16,float16,0,2.45087464650472
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,12,4,128,1,float16,float16,0,2.509023984273275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,12,4,128,1,float16,float16,0,20.242757161458332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,12,1,128,1,float16,float16,0,1.3323466777801514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,12,2,128,1,float16,float16,0,1.3542133967081706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,12,4,128,1,float16,float16,0,1.363807996114095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,12,12,128,1,float16,float16,0,2.89409605662028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,12,1,128,1,float16,float16,0,5.371642430623372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,12,2,128,1,float16,float16,0,5.4376265207926435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,12,4,128,1,float16,float16,0,5.660378774007161
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,12,12,128,1,float16,float16,0,5.756250381469727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,12,1,128,1,float16,float16,0,2.6349973678588867
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,12,12,128,1,float16,float16,0,1.5472639401753743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,12,1,128,1,float16,float16,0,1.4067786534627278
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,12,2,128,1,float16,float16,0,2.8409334818522134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,12,4,128,1,float16,float16,0,2.988885243733724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,12,2,128,1,float16,float16,0,1.4271413485209148
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,12,1,128,1,float16,float16,0,10.946613311767578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,12,2,128,1,float16,float16,0,11.039253234863281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,12,12,128,1,float16,float16,0,0.8309493064880371
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,12,1,128,1,float16,float16,0,0.7648693720499674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,12,4,128,1,float16,float16,0,1.5049066543579102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,12,2,128,1,float16,float16,0,0.8023839791615804
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,12,4,128,1,float16,float16,0,11.757743835449219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,12,4,128,1,float16,float16,0,0.8096053600311279
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,12,12,128,1,float16,float16,0,2.7574612299601235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,12,1,128,1,float16,float16,0,4.817023913065593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,12,2,128,1,float16,float16,0,5.095760027567546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,12,12,128,1,float16,float16,0,5.626181284586589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,12,4,128,1,float16,float16,0,5.409749348958333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,12,1,128,1,float16,float16,0,2.375984032948812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,12,12,128,1,float16,float16,0,1.4303305943806965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,12,2,128,1,float16,float16,0,1.2710293134053547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,12,1,128,1,float16,float16,0,1.250320037206014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,12,2,128,1,float16,float16,0,10.085610707600912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,12,1,128,1,float16,float16,0,9.994778951009115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,12,4,128,1,float16,float16,0,2.713616053263346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,12,2,128,1,float16,float16,0,2.478010654449463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,12,4,128,1,float16,float16,0,10.831045786539713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,12,2,128,1,float16,float16,0,0.3882026672363281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,12,12,128,1,float16,float16,0,0.7479946613311768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,12,12,128,1,float16,float16,0,0.42507731914520264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,12,1,128,1,float16,float16,0,0.6565973361333212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,12,1,128,1,float16,float16,0,0.3680320183436076
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,12,2,128,1,float16,float16,0,0.693674643834432
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,12,4,128,1,float16,float16,0,1.3820907274882
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,12,4,128,1,float16,float16,0,0.7477013270060221
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,12,4,128,1,float16,float16,0,0.4172159830729167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,12,12,128,1,float16,float16,0,1.6840747197469075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,12,1,128,1,float16,float16,0,2.714991887410482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,12,2,128,1,float16,float16,0,2.863306681315104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,12,12,128,1,float16,float16,0,3.2830025355021157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,12,4,128,1,float16,float16,0,3.195317268371582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,12,1,128,1,float16,float16,0,1.398197333017985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,12,12,128,1,float16,float16,0,0.8770773410797119
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,12,1,128,1,float16,float16,0,0.7582186857859293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,12,2,128,1,float16,float16,0,5.985471725463867
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,12,2,128,1,float16,float16,0,1.502639929453532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,12,1,128,1,float16,float16,0,5.606586456298828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,12,4,128,1,float16,float16,0,1.6676799456278484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,12,12,128,1,float16,float16,0,0.475877324740092
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,12,4,128,1,float16,float16,0,6.534240086873372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,12,2,128,1,float16,float16,0,0.7724853356679281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,12,4,128,1,float16,float16,0,0.847599983215332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,12,1,128,1,float16,float16,0,0.43719998995463055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,12,2,128,1,float16,float16,0,0.423311988512675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,12,12,128,1,float16,float16,0,0.27611732482910156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,12,1,128,1,float16,float16,0,0.23334399859110513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,12,2,128,1,float16,float16,0,0.2534240086873372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,12,4,128,1,float16,float16,0,0.48714133103688556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,12,4,128,1,float16,float16,0,0.2624853253364563
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,12,12,128,1,float16,float16,0,1.6789013544718425
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,12,1,128,1,float16,float16,0,2.5730506579081216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,12,4,128,1,float16,float16,0,3.217477480570475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,12,12,128,1,float16,float16,0,3.337535858154297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,12,2,128,1,float16,float16,0,2.7732747395833335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,12,1,128,1,float16,float16,0,1.2966612974802654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,12,2,128,1,float16,float16,0,1.4885120391845703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,12,1,128,1,float16,float16,0,5.257120132446289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,12,4,128,1,float16,float16,0,6.475802739461263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,12,12,128,1,float16,float16,0,0.877013365427653
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,12,4,128,1,float16,float16,0,1.6319467226664226
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,12,1,128,1,float16,float16,0,0.6662240028381348
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,12,2,128,1,float16,float16,0,0.7395679950714111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,12,1,128,1,float16,float16,0,0.36530665556589764
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,12,12,128,1,float16,float16,0,0.47517867883046466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,12,2,128,1,float16,float16,0,5.69268798828125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,12,4,128,1,float16,float16,0,0.4371573527654012
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,12,4,128,1,float16,float16,0,0.8598879973093668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,12,2,128,1,float16,float16,0,0.3904426495234172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,12,12,128,1,float16,float16,0,0.25750933090845746
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,12,1,128,1,float16,float16,0,0.2021920084953308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,12,2,128,1,float16,float16,0,0.22323199113210043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,12,1,128,1,float16,float16,0,0.12770666678746542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,12,2,128,1,float16,float16,0,0.13158933321634927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,12,4,128,1,float16,float16,0,0.14108266433080038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,12,12,128,1,float16,float16,0,0.15373333295186362
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,12,4,128,1,float16,float16,0,0.2587839961051941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,12,1,128,1,float16,float16,0,1.5481600761413574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,12,12,128,1,float16,float16,0,2.111296017964681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,12,2,128,1,float16,float16,0,1.6776746114095051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,12,4,128,1,float16,float16,0,2.111685276031494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,12,1,128,1,float16,float16,0,3.073418617248535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,12,12,128,1,float16,float16,0,1.0779200394948323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,12,4,128,1,float16,float16,0,4.013008117675781
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,12,1,128,1,float16,float16,0,0.7849760055541992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,12,2,128,1,float16,float16,0,0.8617066542307535
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,12,12,128,1,float16,float16,0,0.5688426494598389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,12,1,128,1,float16,float16,0,0.4117973248163859
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,12,4,128,1,float16,float16,0,1.0219626426696777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,12,2,128,1,float16,float16,0,0.45597867170969647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,12,4,128,1,float16,float16,0,0.5280853509902954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,12,1,128,1,float16,float16,0,0.23035732905069986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,12,2,128,1,float16,float16,0,0.2426453431447347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,12,4,128,1,float16,float16,0,0.28277866045633954
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,12,12,128,1,float16,float16,0,0.29890666405359906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,12,12,128,1,float16,float16,0,0.0983733336130778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,12,2,128,1,float16,float16,0,0.13341866930325827
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,12,1,128,1,float16,float16,0,0.12919466694196066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,12,4,128,1,float16,float16,0,0.1553600033124288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,12,12,128,1,float16,float16,0,0.16894932587941489
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,12,1,128,1,float16,float16,0,0.0814933329820633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,12,4,128,1,float16,float16,0,0.09379200140635173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,12,2,128,1,float16,float16,0,0.08528000116348267
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,12,2,128,1,float16,float16,0,3.300618807474772
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,12,12,128,1,float16,float16,0,2.2915306091308594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,12,1,128,1,float16,float16,0,1.5160373051961262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,12,2,128,1,float16,float16,0,1.7031893730163574
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,12,12,128,1,float16,float16,0,1.157866636912028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,12,1,128,1,float16,float16,0,0.7751093705495199
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,12,2,128,1,float16,float16,0,0.8756000200907389
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,12,1,128,1,float16,float16,0,3.045482635498047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,12,12,128,1,float16,float16,0,0.5952266852060953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,12,4,128,1,float16,float16,0,1.0827893416086833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,12,1,128,1,float16,float16,0,0.3989599943161011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,12,12,128,1,float16,float16,0,0.3110719919204712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,12,2,128,1,float16,float16,0,0.45232001940409344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,12,2,128,1,float16,float16,0,0.24209066232045492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,12,4,128,1,float16,float16,0,0.5607253313064575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,12,2,128,1,float16,float16,0,3.4166558583577475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,12,1,128,1,float16,float16,0,0.21711466709772745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,12,4,128,1,float16,float16,0,0.29151999950408936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,12,1,128,1,float16,float16,0,0.12243733803431193
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,12,12,128,1,float16,float16,0,0.1718399922053019
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,12,12,128,1,float16,float16,0,0.10133866469065349
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,12,4,128,1,float16,float16,0,2.1291893323262534
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,12,2,128,1,float16,float16,0,0.13405866424242655
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,12,2,128,1,float16,float16,0,0.07811200122038524
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,12,12,128,1,float16,float16,0,0.05153599878152212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,12,1,128,1,float16,float16,0,0.07580799857775371
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,12,1,128,1,float16,float16,0,0.04558399816354116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,12,4,128,1,float16,float16,0,0.08790399630864461
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,12,4,128,1,float16,float16,0,4.246351877848308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,12,4,128,1,float16,float16,0,0.05034666756788889
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,12,4,128,1,float16,float16,0,0.16208533445994058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,12,2,128,1,float16,float16,0,0.046629334489504494
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,12,12,128,1,float16,float16,0,1.7676746050516765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,12,2,128,1,float16,float16,0,1.1926346619923909
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,12,1,128,1,float16,float16,0,1.9982239405314128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,12,12,128,1,float16,float16,0,0.8909333546956381
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,12,1,128,1,float16,float16,0,0.5156853199005127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,12,4,128,1,float16,float16,0,1.6268587112426758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,12,1,128,1,float16,float16,0,1.0142453511555989
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,12,12,128,1,float16,float16,0,0.4574720064798991
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,12,2,128,1,float16,float16,0,0.6213493347167969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,12,1,128,1,float16,float16,0,0.2661173343658447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,12,4,128,1,float16,float16,0,0.8248480161031088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,12,12,128,1,float16,float16,0,0.24245333671569824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,12,2,128,1,float16,float16,0,0.171450674533844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,12,4,128,1,float16,float16,0,0.4238773187001546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,12,12,128,1,float16,float16,0,0.13102933764457703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,12,1,128,1,float16,float16,0,0.14518400033315024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,12,1,128,1,float16,float16,0,0.08305599788824718
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,12,2,128,1,float16,float16,0,0.09599467118581136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,12,1,128,1,float16,float16,0,0.04942933221658071
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,12,12,128,1,float16,float16,0,0.0765226682027181
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,12,4,128,1,float16,float16,0,0.1202826698621114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,12,4,128,1,float16,float16,0,0.22211732467015585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,12,2,128,1,float16,float16,0,2.3466933568318686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,12,2,128,1,float16,float16,0,0.3219199975331624
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,12,2,128,1,float16,float16,0,0.053029333551724754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,12,4,128,1,float16,float16,0,0.06247466802597046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,12,12,128,1,float16,float16,0,0.0351946676770846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,12,1,128,1,float16,float16,0,0.02874133239189784
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,12,4,128,1,float16,float16,0,0.034341332813103996
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,12,12,128,1,float16,float16,0,0.025493333737055462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,12,4,128,1,float16,float16,0,3.222544034322103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,12,1,128,1,float16,float16,0,0.022170667846997578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,12,2,128,1,float16,float16,0,0.023589332898457844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,12,4,128,1,float16,float16,0,0.025461333493391674
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,12,2,128,1,float16,float16,0,0.030394665896892548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,12,1,128,1,float16,float16,0,0.38277868429819745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,12,2,128,1,float16,float16,0,0.4840586582819621
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,12,4,128,1,float16,float16,0,1.3777440388997395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,12,4,128,1,float16,float16,0,0.6889386971791586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,12,1,128,1,float16,float16,0,0.19478933016459146
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,12,12,128,1,float16,float16,0,0.3893440167109172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,12,12,128,1,float16,float16,0,0.2050186594327291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,12,1,128,1,float16,float16,0,0.7603092988332113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,12,12,128,1,float16,float16,0,0.7613653341929117
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,12,2,128,1,float16,float16,0,0.938378651936849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,12,4,128,1,float16,float16,0,0.35207998752593994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,12,2,128,1,float16,float16,0,0.13422399759292603
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,12,2,128,1,float16,float16,0,0.2511413296063741
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,12,4,128,1,float16,float16,0,0.18475200732549033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,12,12,128,1,float16,float16,0,0.06205333272616068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,12,1,128,1,float16,float16,0,0.10697066783905029
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,12,4,128,1,float16,float16,0,0.09873599807421367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,12,1,128,1,float16,float16,0,0.03433600068092346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,12,2,128,1,float16,float16,0,0.03902400036652883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,12,4,128,1,float16,float16,0,0.04842666784922282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,12,12,128,1,float16,float16,0,0.02941333254178365
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,12,2,128,1,float16,float16,0,0.02458133300145467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,12,1,128,1,float16,float16,0,0.023007998863856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,12,4,128,1,float16,float16,0,0.028175999720891316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,12,12,128,1,float16,float16,0,0.020106667031844456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,12,4,128,1,float16,float16,0,0.01995733380317688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,12,12,128,1,float16,float16,0,0.017488000293572743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,12,2,128,1,float16,float16,0,0.01798933371901512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,12,1,128,1,float16,float16,0,0.015669333438078564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,12,1,128,1,float16,float16,0,0.0164533331990242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,12,2,128,1,float16,float16,0,0.01613866661985715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,12,4,128,1,float16,float16,0,0.01762666677435239
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,12,2,128,1,float16,float16,0,0.07384000221888225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,12,1,128,1,float16,float16,0,0.059605335195859276
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,12,1,128,1,float16,float16,0,0.1572106679280599
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,12,12,128,1,float16,float16,0,0.11146666606267293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,12,12,128,1,float16,float16,0,0.1886026660601298
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,12,2,128,1,float16,float16,0,0.21228800217310587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,12,12,128,1,float16,float16,0,0.3670080105463664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,12,1,128,1,float16,float16,0,0.3440106709798177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,12,4,128,1,float16,float16,0,0.321615993976593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,12,1,128,1,float16,float16,0,0.08683199683825175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,12,2,128,1,float16,float16,0,0.11212799946467082
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,12,4,128,1,float16,float16,0,0.16670932372411093
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,12,4,128,1,float16,float16,0,0.08969066540400188
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,12,2,128,1,float16,float16,0,0.44571733474731445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,12,12,128,1,float16,float16,0,0.05529599885145823
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,12,4,128,1,float16,float16,0,0.6511040131251017
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,12,2,128,1,float16,float16,0,0.0313226655125618
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,12,2,128,1,float16,float16,0,0.06313600142796834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,12,12,128,1,float16,float16,0,0.10346666971842448
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,12,1,128,1,float16,float16,0,0.027866666515668232
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,12,12,128,1,float16,float16,0,0.02441066751877467
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,12,1,128,1,float16,float16,0,0.04827733337879181
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,12,2,128,1,float16,float16,0,0.019637333850065868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,12,4,128,1,float16,float16,0,0.023285334308942158
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,12,2,128,1,float16,float16,0,0.014954666296641031
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,12,1,128,1,float16,float16,0,0.01320533330241839
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,12,2,128,1,float16,float16,0,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,12,4,128,1,float16,float16,0,0.0169813334941864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,12,12,128,1,float16,float16,0,0.014592000593741735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,12,12,128,1,float16,float16,0,0.01708799973130226
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,12,4,128,1,float16,float16,0,0.01431999976436297
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,12,1,128,1,float16,float16,0,0.012416000167528788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,12,12,128,1,float16,float16,0,0.012639999389648438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,12,1,128,1,float16,float16,0,0.012234666695197424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,12,2,128,1,float16,float16,0,0.01221866657336553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,12,4,128,1,float16,float16,0,0.01259200026591619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,12,4,128,1,float16,float16,0,0.040192000567913055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,12,1,128,1,float16,float16,0,0.017903999735911686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,12,1,128,1,float16,float16,0,0.08591467142105103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,12,4,128,1,float16,float16,0,0.16532267133394876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,12,12,128,1,float16,float16,0,0.10257066289583842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,12,12,128,1,float16,float16,0,0.18841065963109335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,12,2,128,1,float16,float16,0,0.11283199985822041
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,12,1,128,1,float16,float16,0,0.15442132949829102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,12,1,128,1,float16,float16,0,0.04788800080617269
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,12,2,128,1,float16,float16,0,0.21082133054733276
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,12,1,128,1,float16,float16,0,0.02677333354949951
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,12,2,128,1,float16,float16,0,0.06282133360703786
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,12,4,128,1,float16,float16,0,0.31752532720565796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,12,4,128,1,float16,float16,0,0.08957333366076152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,12,12,128,1,float16,float16,0,0.05503466725349426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,12,4,128,1,float16,float16,0,0.039493332306543984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,12,2,128,1,float16,float16,0,0.03054400036732356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,12,1,128,1,float16,float16,0,0.017658667018016178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,12,2,128,1,float16,float16,0,0.01934933289885521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,12,4,128,1,float16,float16,0,0.023237332701683044
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,12,1,128,1,float16,float16,0,0.013178666432698568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,12,2,128,1,float16,float16,0,0.01458666721979777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,12,4,128,1,float16,float16,0,0.016517333686351776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,12,12,128,1,float16,float16,0,0.02409599969784419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,12,12,128,1,float16,float16,0,0.014085333794355392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,12,12,128,1,float16,float16,0,0.016688000410795212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,12,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,12,12,128,1,float16,float16,0,0.012666666259368261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,12,1,128,1,float16,float16,0,0.012144000579913458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,12,4,128,1,float16,float16,0,0.014266667266686758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,12,2,128,1,float16,float16,0,0.01219733307758967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,12,4,128,1,float16,float16,0,0.01240533341964086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,12,1,128,1,float16,float16,0,0.012202666451533636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,12,12,128,1,float16,float16,0,0.012069333344697952
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,12,1,128,1,float16,float16,0,0.011877333124478659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,12,4,128,1,float16,float16,0,0.012309333930412928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,12,2,128,1,float16,float16,0,0.011839999506870905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,12,1,128,1,float16,float16,0,0.07172266642252605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,12,2,128,1,float16,float16,0,0.08565333485603333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,12,4,128,1,float16,float16,0,0.11100799838701884
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,12,12,128,1,float16,float16,0,0.12032000223795573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,12,12,128,1,float16,float16,0,0.06565866867701213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,12,1,128,1,float16,float16,0,0.13476266463597616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,12,1,128,1,float16,float16,0,0.03806933263937632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,12,2,128,1,float16,float16,0,0.15827199816703796
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,12,2,128,1,float16,float16,0,0.04179200033346812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,12,4,128,1,float16,float16,0,0.21022399266560873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,12,4,128,1,float16,float16,0,0.0518506666024526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,12,12,128,1,float16,float16,0,0.030405332644780476
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,12,1,128,1,float16,float16,0,0.023578666150569916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,12,2,128,1,float16,float16,0,0.025279998779296875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,12,12,128,1,float16,float16,0,0.018730666488409042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,12,4,128,1,float16,float16,0,0.029167999823888142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,12,1,128,1,float16,float16,0,0.015072000523408255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,12,12,128,1,float16,float16,0,0.014090667168299357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,12,1,128,1,float16,float16,0,0.012149333953857422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,12,4,128,1,float16,float16,0,0.018650667121013004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,12,2,128,1,float16,float16,0,0.016741332908471424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,12,4,128,1,float16,float16,0,0.014074667046467463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,12,2,128,1,float16,float16,0,0.01258133351802826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,12,12,128,1,float16,float16,0,0.012522666404644648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,12,1,128,1,float16,float16,0,0.011760000139474869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,12,2,128,1,float16,float16,0,0.011839999506870905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,12,12,128,1,float16,float16,0,0.011930666863918304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,12,1,128,1,float16,float16,0,0.01156266654531161
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,12,4,128,1,float16,float16,0,0.012181332955757776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,12,12,128,1,float16,float16,0,0.011674666156371435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,12,4,128,1,float16,float16,0,0.011893333246310553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,12,2,128,1,float16,float16,0,0.011690666278203329
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,12,1,128,1,float16,float16,0,0.011509332805871964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,12,2,128,1,float16,float16,0,0.011488000551859537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,12,4,128,1,float16,float16,0,0.011578666667143503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,12,1,128,1,float16,float16,0,0.061466669042905174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,12,4,128,1,float16,float16,0,0.07400533556938171
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,12,2,128,1,float16,float16,0,0.06509866813818614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,12,12,128,1,float16,float16,0,0.0904960036277771
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,12,12,128,1,float16,float16,0,0.04307200014591217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,12,1,128,1,float16,float16,0,0.035802667339642845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,12,2,128,1,float16,float16,0,0.037690666814645134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,12,2,128,1,float16,float16,0,0.1346560021241506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,12,1,128,1,float16,float16,0,0.12069333593050639
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,12,4,128,1,float16,float16,0,0.041690667470296226
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,12,4,128,1,float16,float16,0,0.1605226695537567
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,12,12,128,1,float16,float16,0,0.025487999121348064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,12,2,128,1,float16,float16,0,0.02348266790310542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,12,1,128,1,float16,float16,0,0.02176533391078313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,12,1,128,1,float16,float16,0,0.014661333213249842
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,12,4,128,1,float16,float16,0,0.02532266577084859
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,12,2,128,1,float16,float16,0,0.014773332824309668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,12,12,128,1,float16,float16,0,0.016986666868130367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,12,1,128,1,float16,float16,0,0.011898666620254517
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,12,4,128,1,float16,float16,0,0.016901332885026932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,12,2,128,1,float16,float16,0,0.012122667084137598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,12,12,128,1,float16,float16,0,0.012624000509579977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,12,4,128,1,float16,float16,0,0.012506666282812754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,12,12,128,1,float16,float16,0,0.012042666474978128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,12,2,128,1,float16,float16,0,0.011461333682139715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,12,1,128,1,float16,float16,0,0.011648000528415045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,12,4,128,1,float16,float16,0,0.011781333635250727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,12,12,128,1,float16,float16,0,0.01191466674208641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,12,1,128,1,float16,float16,0,0.011567999919255575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,12,2,128,1,float16,float16,0,0.011626667032639185
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,12,4,128,1,float16,float16,0,0.01180800050497055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,12,12,128,1,float16,float16,0,0.011584000041087469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,12,1,128,1,float16,float16,0,0.011711999773979187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,12,4,128,1,float16,float16,0,0.011498666057984034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,12,2,128,1,float16,float16,0,0.011498666057984034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,12,12,128,1,float16,float16,0,0.03356266766786575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,12,12,128,1,float16,float16,0,0.056741332014401756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,0,0.01602666700879733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,0,0.03379199902216593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,0,0.02716800073782603
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,0,0.013210666676362356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,12,12,128,1,float16,float16,0,0.020997333029905956
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,0,0.013077333569526672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,0,0.022250667214393616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,0,0.026895999908447266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,0,0.02083733429511388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,0,0.03498666733503342
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,0,0.01964266722400983
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,0,0.05277333160241445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,0,0.022245332598686218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,12,12,128,1,float16,float16,0,0.014175999909639359
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,0,0.019146667172511418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,12,12,128,1,float16,float16,0,0.011370666325092316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,0,0.019765333582957584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,0,0.01854933301607768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,0,0.012426666915416718
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,0,0.018906666586796444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,12,12,128,1,float16,float16,0,0.011242666592200598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,0,0.018357332795858383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,0,0.01259200026591619
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,12,12,128,1,float16,float16,0,0.011087999989589056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,0,0.018320000420014065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,0,0.018415999909241993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,0,0.012437333663304647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,0,0.018538666268189747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,12,12,128,1,float16,float16,0,0.010944000134865442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,0,0.012063999970753988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,0,0.012719999998807907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,0,0.012554666648308435
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,1,128,1,float16,float16,0,12.591397603352865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,2,128,1,float16,float16,0,12.672991434733072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,4,128,1,float16,float16,0,12.970965067545572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,float16,0,13.099589029947916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,1,128,1,float16,float16,0,6.147002538045247
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,2,128,1,float16,float16,0,6.206736246744792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,float16,0,6.568464279174805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,4,128,1,float16,float16,0,6.346954981486003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,8,1,128,1,float16,float16,0,24.843739827473957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,float16,0,3.2833919525146484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,8,4,128,1,float16,float16,0,25.86730702718099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,8,2,128,1,float16,float16,0,24.998822530110676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,float16,0,13.118746439615885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,1,128,1,float16,float16,0,3.114805221557617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,4,128,1,float16,float16,0,3.3299573262532554
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,2,128,1,float16,float16,0,3.1616694132486978
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,float16,0,3.3152106602986655
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,float16,0,6.376746495564778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,float16,0,7.581189473470052
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,1,128,1,float16,float16,0,7.033936182657878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,2,128,1,float16,float16,0,6.924421310424805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,4,128,1,float16,float16,0,7.399530410766602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,float16,0,7.4497330983479815
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,8,1,128,1,float16,float16,0,14.111930847167969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,float16,0,3.7830985387166343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,1,128,1,float16,float16,0,3.605408032735189
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,2,128,1,float16,float16,0,3.605722745259603
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,8,2,128,1,float16,float16,0,14.345690409342447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,4,128,1,float16,float16,0,3.7693920135498047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,8,4,128,1,float16,float16,0,15.097722371419271
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,float16,0,3.737680117289225
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,2,128,1,float16,float16,0,1.893130620320638
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,1,128,1,float16,float16,0,1.9308959643046062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,float16,0,2.067824045817057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,4,128,1,float16,float16,0,1.9968959490458171
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,float16,0,2.0287413597106934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,1,128,1,float16,float16,0,4.881493250528972
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,2,128,1,float16,float16,0,5.011343955993652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,float16,0,5.27890141805013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,4,128,1,float16,float16,0,5.126890818277995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,float16,0,5.273626645406087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,1,128,1,float16,float16,0,2.5130720138549805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,float16,0,2.746117273966471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,2,128,1,float16,float16,0,2.5508960088094077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,8,1,128,1,float16,float16,0,9.948005040486654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,4,128,1,float16,float16,0,2.6721973419189453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,8,2,128,1,float16,float16,0,10.204847971598307
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,float16,0,2.738314628601074
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,8,4,128,1,float16,float16,0,10.415162404378256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,float16,0,1.4471786816914876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,1,128,1,float16,float16,0,1.3605012893676758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,2,128,1,float16,float16,0,1.3665013313293457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,4,128,1,float16,float16,0,1.5465332667032878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,float16,0,1.444543997446696
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,1,128,1,float16,float16,0,6.239285151163737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,2,128,1,float16,float16,0,6.466005325317383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,float16,0,6.980837504069011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,4,128,1,float16,float16,0,6.893039703369141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,float16,0,6.8437652587890625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,float16,0,3.5345226923624673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,1,128,1,float16,float16,0,3.1859572728474936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,2,128,1,float16,float16,0,3.2875467936197915
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,4,128,1,float16,float16,0,3.4708000818888345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,8,1,128,1,float16,float16,0,13.022090911865234
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,8,2,128,1,float16,float16,0,13.000784556070963
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,float16,0,3.53653875986735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,1,128,1,float16,float16,0,1.6422665913899739
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,float16,0,1.8363572756449382
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,2,128,1,float16,float16,0,1.7237226168314617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,8,4,128,1,float16,float16,0,13.720731099446615
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,float16,0,0.9912693500518799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,1,128,1,float16,float16,0,0.9064533710479736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,4,128,1,float16,float16,0,1.8128479321797688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,float16,0,1.934127966562907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,2,128,1,float16,float16,0,0.9173119862874349
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,4,128,1,float16,float16,0,0.9795413017272949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,float16,0,0.980293353398641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,1,128,1,float16,float16,0,3.48305606842041
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,float16,0,4.044005393981934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,2,128,1,float16,float16,0,3.666293462117513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,4,128,1,float16,float16,0,3.981712023417155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,8,1,128,1,float16,float16,0,7.114095687866211
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,8,2,128,1,float16,float16,0,7.490026473999023
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,float16,0,2.081109364827474
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,1,128,1,float16,float16,0,1.8025120099385579
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,2,128,1,float16,float16,0,1.912218729654948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,8,4,128,1,float16,float16,0,7.978000005086263
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,1,128,1,float16,float16,0,0.9621600310007731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,float16,0,1.0903626283009846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,2,128,1,float16,float16,0,1.036346673965454
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,float16,0,4.08897082010905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,4,128,1,float16,float16,0,2.120858669281006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,float16,0,2.102245330810547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,4,128,1,float16,float16,0,1.096490701039632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,float16,0,0.6125119924545288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,1,128,1,float16,float16,0,0.548314650853475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,2,128,1,float16,float16,0,0.5933173497517904
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,float16,0,1.0756906668345134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,4,128,1,float16,float16,0,0.6134133338928223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,float16,0,0.6074133316675822
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,2,128,1,float16,float16,0,3.397109349568685
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,1,128,1,float16,float16,0,3.2179253896077475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,float16,0,3.882346789042155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,4,128,1,float16,float16,0,3.805210749308268
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,1,128,1,float16,float16,0,1.680565357208252
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,float16,0,2.0029172897338867
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,8,1,128,1,float16,float16,0,6.48695437113444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,2,128,1,float16,float16,0,1.771183967590332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,8,2,128,1,float16,float16,0,7.082799911499023
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,4,128,1,float16,float16,0,1.9583306312561035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,8,4,128,1,float16,float16,0,7.762970606486003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,float16,0,1.0267039934794109
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,2,128,1,float16,float16,0,0.9156320095062256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,float16,0,1.0315732955932617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,4,128,1,float16,float16,0,1.0595520337422688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,float16,0,0.5829439957936605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,1,128,1,float16,float16,0,0.8814773559570312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,1,128,1,float16,float16,0,0.47522131601969403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,float16,0,1.9642933209737141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,2,128,1,float16,float16,0,0.4914933443069458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,4,128,1,float16,float16,0,0.5570453405380249
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,float16,0,0.5794506470362345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,float16,0,0.33083732922871906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,4,128,1,float16,float16,0,0.31602134307225543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,1,128,1,float16,float16,0,0.2741653323173523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,2,128,1,float16,float16,0,0.2842026750246684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,float16,0,0.3224853277206421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,float16,0,3.8090718587239585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,2,128,1,float16,float16,0,2.022864023844401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,1,128,1,float16,float16,0,1.8652267456054688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,float16,0,2.370314598083496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,4,128,1,float16,float16,0,2.3360800743103027
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,float16,0,2.3740053176879883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,1,128,1,float16,float16,0,0.9647786617279053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,float16,0,1.2072800000508626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,8,2,128,1,float16,float16,0,4.030319849650065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,2,128,1,float16,float16,0,1.0591039657592773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,8,1,128,1,float16,float16,0,3.7549546559651694
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,4,128,1,float16,float16,0,1.198533296585083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,8,4,128,1,float16,float16,0,4.678720156351726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,1,128,1,float16,float16,0,0.5107413530349731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,2,128,1,float16,float16,0,0.5920960108439127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,float16,0,0.35077865918477374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,4,128,1,float16,float16,0,0.638431986172994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,float16,0,0.6391520102818807
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,1,128,1,float16,float16,0,0.30188800891240436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,2,128,1,float16,float16,0,0.3113226691881816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,float16,0,0.6749546527862549
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,float16,0,1.2196053663889568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,float16,0,0.20016000668207803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,4,128,1,float16,float16,0,0.3593013286590576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,float16,0,0.3542720079421997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,1,128,1,float16,float16,0,0.16617066661516824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,2,128,1,float16,float16,0,0.1792800029118856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,4,128,1,float16,float16,0,0.19097065925598145
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,float16,0,0.2156426707903544
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,float16,0,2.411743958791097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,4,128,1,float16,float16,0,2.371002674102783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,float16,0,2.420191923777262
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,1,128,1,float16,float16,0,1.7585919698079426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,2,128,1,float16,float16,0,1.95632537206014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,float16,0,1.2281653086344402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,1,128,1,float16,float16,0,0.9004480044047037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,8,1,128,1,float16,float16,0,3.5471200942993164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,8,4,128,1,float16,float16,0,4.767290751139323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,2,128,1,float16,float16,0,1.0175200303395588
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,4,128,1,float16,float16,0,1.206938664118449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,1,128,1,float16,float16,0,0.4763253529866536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,8,2,128,1,float16,float16,0,3.867936134338379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,float16,0,1.246234655380249
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,float16,0,0.6436426639556885
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,4,128,1,float16,float16,0,0.6406826575597128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,2,128,1,float16,float16,0,0.5430026849110922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,float16,0,0.6493599812189738
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,1,128,1,float16,float16,0,0.262447992960612
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,float16,0,0.34958934783935547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,4,128,1,float16,float16,0,0.338922659556071
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,float16,0,0.34513600667317706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,float16,0,0.2002613345781962
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,2,128,1,float16,float16,0,0.15968533356984457
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,1,128,1,float16,float16,0,0.15110933780670166
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,float16,0,0.11230933666229248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,4,128,1,float16,float16,0,0.19132266441980997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,2,128,1,float16,float16,0,0.299621323744456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,float16,0,0.20122667153676352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,1,128,1,float16,float16,0,0.09711466232935588
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,2,128,1,float16,float16,0,0.10002133250236511
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,4,128,1,float16,float16,0,0.10917333761850993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,float16,0,0.11209066708882649
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,float16,0,1.5706186294555664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,1,128,1,float16,float16,0,1.0643093585968018
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,4,128,1,float16,float16,0,1.5250879923502605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,2,128,1,float16,float16,0,1.2159146467844646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,8,4,128,1,float16,float16,0,3.021754582722982
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,8,1,128,1,float16,float16,0,2.0919413566589355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,1,128,1,float16,float16,0,0.5479466517766317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,float16,0,0.7975467046101888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,2,128,1,float16,float16,0,0.6303893327713013
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,8,2,128,1,float16,float16,0,2.37555726369222
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,float16,0,1.5711253484090169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,float16,0,0.4191360076268514
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,1,128,1,float16,float16,0,0.29499733448028564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,4,128,1,float16,float16,0,0.7847039699554443
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,2,128,1,float16,float16,0,0.3352959950764974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,float16,0,0.4242719809214274
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,float16,0,0.2290239930152893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,1,128,1,float16,float16,0,0.1646773318449656
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,float16,0,0.7951947053273519
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,2,128,1,float16,float16,0,0.18844799200693765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,1,128,1,float16,float16,0,0.12575999895731607
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,4,128,1,float16,float16,0,0.41493332386016846
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,float16,0,0.16553599635759988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,float16,0,0.2477653423945109
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,2,128,1,float16,float16,0,0.10707733035087585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,4,128,1,float16,float16,0,0.12297067046165466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,4,128,1,float16,float16,0,0.22201067209243774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,float16,0,0.06934399902820587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,float16,0,0.13342932860056558
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,1,128,1,float16,float16,0,0.059343998630841575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,2,128,1,float16,float16,0,0.06141333281993866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,float16,0,0.06905599931875865
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,4,128,1,float16,float16,0,0.06808533271153767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,float16,0,1.7423680623372395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,1,128,1,float16,float16,0,1.0630773703257244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,2,128,1,float16,float16,0,1.2455946604410808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,8,1,128,1,float16,float16,0,2.0973973274230957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,float16,0,0.8799146811167399
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,1,128,1,float16,float16,0,0.5453653335571289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,float16,0,1.7249280611673992
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,2,128,1,float16,float16,0,0.6481546560923258
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,4,128,1,float16,float16,0,1.6700746218363445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,4,128,1,float16,float16,0,0.8555733362833658
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,8,2,128,1,float16,float16,0,2.424565315246582
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,float16,0,0.8785333633422852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,1,128,1,float16,float16,0,0.2869439919789632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,2,128,1,float16,float16,0,0.34060267607371014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,float16,0,0.45391468207041424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,1,128,1,float16,float16,0,0.1588266690572103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,8,4,128,1,float16,float16,0,3.330629348754883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,float16,0,0.4606773455937703
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,2,128,1,float16,float16,0,0.18584533532460532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,4,128,1,float16,float16,0,0.23568000396092734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,float16,0,0.24372265736262003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,float16,0,0.13738666971524557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,4,128,1,float16,float16,0,0.4419039885203044
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,float16,0,0.24262400468190512
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,2,128,1,float16,float16,0,0.0995093286037445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,4,128,1,float16,float16,0,0.13115200400352478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,float16,0,0.07175466914971669
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,1,128,1,float16,float16,0,0.05646933118502299
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,float16,0,0.1368053356806437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,float16,0,0.0719946672519048
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,4,128,1,float16,float16,0,0.06832533578077953
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,float16,0,0.05027199784914652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,2,128,1,float16,float16,0,0.045925334095954895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,1,128,1,float16,float16,0,0.04420266548792521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,4,128,1,float16,float16,0,0.05004266897837321
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,2,128,1,float16,float16,0,0.05958933134873708
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,float16,0,0.05026666820049286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,1,128,1,float16,float16,0,0.09156800309816997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,1,128,1,float16,float16,0,0.7240266799926758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,2,128,1,float16,float16,0,0.9102506637573242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,float16,0,0.7050027052561442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,4,128,1,float16,float16,0,1.344154675801595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,1,128,1,float16,float16,0,0.37171733379364014
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,8,1,128,1,float16,float16,0,1.4188106854756672
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,2,128,1,float16,float16,0,0.4750186602274577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,float16,0,1.361738681793213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,4,128,1,float16,float16,0,0.6830293337504069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,8,2,128,1,float16,float16,0,1.7795519828796387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,1,128,1,float16,float16,0,0.19559999306996664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,float16,0,0.19413334131240845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,float16,0,0.7062346935272217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,float16,0,1.366447925567627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,4,128,1,float16,float16,0,0.35285866260528564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,float16,0,0.3609813451766968
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,1,128,1,float16,float16,0,0.10920000076293945
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,8,4,128,1,float16,float16,0,2.638869285583496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,4,128,1,float16,float16,0,0.1869600017865499
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,float16,0,0.10769066214561462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,4,128,1,float16,float16,0,0.10159466663996379
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,2,128,1,float16,float16,0,0.06986666719118755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,float16,0,0.19283199310302734
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,float16,0,0.056559999783833824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,float16,0,0.10801600416501363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,2,128,1,float16,float16,0,0.04326933125654856
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,1,128,1,float16,float16,0,0.04045866678158442
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,4,128,1,float16,float16,0,0.05258133510748545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,float16,0,0.05666666726271311
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,float16,0,0.36441067854563397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,2,128,1,float16,float16,0,0.25082133213679
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,float16,0,0.03435733417669932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,1,128,1,float16,float16,0,0.028773332635561626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,2,128,1,float16,float16,0,0.030229332546393078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,float16,0,0.025226667523384094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,1,128,1,float16,float16,0,0.021744000415007275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,2,128,1,float16,float16,0,0.02311466634273529
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,4,128,1,float16,float16,0,0.025040000677108765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,float16,0,0.025402667621771496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,float16,0,0.034458667039871216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,4,128,1,float16,float16,0,0.03379733363787333
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,2,128,1,float16,float16,0,0.13570666313171387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,1,128,1,float16,float16,0,0.28229333957036334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,2,128,1,float16,float16,0,0.3866613308588664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,8,1,128,1,float16,float16,0,0.5554240147272745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,1,128,1,float16,float16,0,0.060047999024391174
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,float16,0,0.6159253517786661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,8,2,128,1,float16,float16,0,0.7408533096313477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,4,128,1,float16,float16,0,0.5884693463643392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,2,128,1,float16,float16,0,0.19876799980799356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,float16,0,0.3141653339068095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,8,4,128,1,float16,float16,0,1.1663413047790527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,1,128,1,float16,float16,0,0.14589866995811462
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,float16,0,0.6170560121536255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,1,128,1,float16,float16,0,0.08210133512814839
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,float16,0,0.16738667090733847
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,4,128,1,float16,float16,0,0.3019733428955078
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,float16,0,0.31296000878016156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,1,128,1,float16,float16,0,0.04461866617202759
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,float16,0,0.1686720053354899
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,float16,0,0.04520533482233683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,2,128,1,float16,float16,0,0.10827199618021648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,2,128,1,float16,float16,0,0.05388266841570536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,4,128,1,float16,float16,0,0.08649067083994548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,2,128,1,float16,float16,0,0.03204799940188726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,float16,0,0.04494399825731913
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,1,128,1,float16,float16,0,0.028565332293510437
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,4,128,1,float16,float16,0,0.04033066580692927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,float16,0,0.09258666634559631
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,4,128,1,float16,float16,0,0.028037334481875103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,1,128,1,float16,float16,0,0.02253866692384084
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,float16,0,0.02849599967400233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,2,128,1,float16,float16,0,0.02409599969784419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,float16,0,0.02863999952872594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,1,128,1,float16,float16,0,0.016074666132529575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,2,128,1,float16,float16,0,0.017397332936525345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,float16,0,0.09274133046468098
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,float16,0,0.01966399947802226
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,4,128,1,float16,float16,0,0.01950399950146675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,float16,0,0.019679999599854153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,float16,0,0.01820266619324684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,1,128,1,float16,float16,0,0.01525866612792015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,4,128,1,float16,float16,0,0.15922666589419046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,2,128,1,float16,float16,0,0.015717333803574245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,float16,0,0.017488000293572743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,4,128,1,float16,float16,0,0.017397332936525345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,1,128,1,float16,float16,0,0.11971732974052429
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,2,128,1,float16,float16,0,0.1731839974721273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,float16,0,0.1570186714331309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,4,128,1,float16,float16,0,0.2802026669184367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,float16,0,0.29846400022506714
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,float16,0,0.29862932364145917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,1,128,1,float16,float16,0,0.06744533280531566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,8,1,128,1,float16,float16,0,0.251306672890981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,2,128,1,float16,float16,0,0.09479999542236328
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,8,2,128,1,float16,float16,0,0.3543359835942586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,1,128,1,float16,float16,0,0.035504000882307686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,4,128,1,float16,float16,0,0.14683199922243753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,8,4,128,1,float16,float16,0,0.5589280128479004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,float16,0,0.1574133336544037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,float16,0,0.08590933680534363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,4,128,1,float16,float16,0,0.07781333227952321
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,float16,0,0.03979199876387914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,2,128,1,float16,float16,0,0.04534400006135305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,1,128,1,float16,float16,0,0.024314666787783306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,float16,0,0.023423999547958374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,float16,0,0.03908800085385641
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,1,128,1,float16,float16,0,0.017397332936525345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,4,128,1,float16,float16,0,0.023061332603295643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,2,128,1,float16,float16,0,0.019018666197856266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,float16,0,0.08656533559163411
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,4,128,1,float16,float16,0,0.03642133375008901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,float16,0,0.0233599990606308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,float16,0,0.016597333053747814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,2,128,1,float16,float16,0,0.0143306665122509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,4,128,1,float16,float16,0,0.016202667107184727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,1,128,1,float16,float16,0,0.012847999731699625
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,float16,0,0.014314666390419006
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,1,128,1,float16,float16,0,0.012400000045696894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,float16,0,0.016352000335852306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,2,128,1,float16,float16,0,0.027717334528764088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,2,128,1,float16,float16,0,0.012416000167528788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,4,128,1,float16,float16,0,0.0141546664138635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,float16,0,0.012719999998807907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,float16,0,0.01440000037352244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,1,128,1,float16,float16,0,0.012186666329701742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,2,128,1,float16,float16,0,0.012063999970753988
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,4,128,1,float16,float16,0,0.012309333930412928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,float16,0,0.012655999511480331
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,1,128,1,float16,float16,0,0.06808533271153767
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,2,128,1,float16,float16,0,0.09220799803733826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,4,128,1,float16,float16,0,0.14617600043614706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,float16,0,0.15566933155059814
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,float16,0,0.08507200082143147
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,8,1,128,1,float16,float16,0,0.11914133032162984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,8,2,128,1,float16,float16,0,0.1716960072517395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,1,128,1,float16,float16,0,0.034458667039871216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,2,128,1,float16,float16,0,0.04398400088151296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,float16,0,0.154831995566686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,1,128,1,float16,float16,0,0.023936000963052113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,2,128,1,float16,float16,0,0.027429332335789997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,float16,0,0.03995199998219808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,8,4,128,1,float16,float16,0,0.2774239977200826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,float16,0,0.0860533316930135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,4,128,1,float16,float16,0,0.07960000137488048
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,float16,0,0.02317333221435547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,1,128,1,float16,float16,0,0.017312000195185345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,float16,0,0.03969600051641464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,4,128,1,float16,float16,0,0.03590933233499527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,2,128,1,float16,float16,0,0.019093333433071773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,4,128,1,float16,float16,0,0.023045333723227184
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,float16,0,0.01648533344268799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,1,128,1,float16,float16,0,0.012794667234023413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,4,128,1,float16,float16,0,0.01621866722901662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,2,128,1,float16,float16,0,0.014271999398867289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,float16,0,0.023082666099071503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,float16,0,0.01642666632930438
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,1,128,1,float16,float16,0,0.012400000045696894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,2,128,1,float16,float16,0,0.012506666282812754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,float16,0,0.014186666657527288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,4,128,1,float16,float16,0,0.014042666802803675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,float16,0,0.014181333283583323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,1,128,1,float16,float16,0,0.011973333855470022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,2,128,1,float16,float16,0,0.01201066623131434
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,float16,0,0.012389333297808966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,4,128,1,float16,float16,0,0.012416000167528788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,float16,0,0.011893333246310553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,1,128,1,float16,float16,0,0.011717333147923151
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,float16,0,0.012346666306257248
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,2,128,1,float16,float16,0,0.011711999773979187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,4,128,1,float16,float16,0,0.011962667107582092
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,float16,0,0.011877333124478659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,float16,0,0.09716266393661499
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,2,128,1,float16,float16,0,0.059936001896858215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,1,128,1,float16,float16,0,0.04881600042184194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,4,128,1,float16,float16,0,0.09076799949010213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,8,1,128,1,float16,float16,0,0.09936533371607463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,8,2,128,1,float16,float16,0,0.12444266676902771
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,2,128,1,float16,float16,0,0.0329066663980484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,1,128,1,float16,float16,0,0.029365333418051403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,float16,0,0.04539200166861216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,8,4,128,1,float16,float16,0,0.17548267046610513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,4,128,1,float16,float16,0,0.04147200038035711
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,float16,0,0.04695466657479604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,2,128,1,float16,float16,0,0.02279466638962428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,float16,0,0.018581333259741466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,1,128,1,float16,float16,0,0.020975999534130096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,float16,0,0.09857599933942159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,float16,0,0.027263998985290527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,4,128,1,float16,float16,0,0.02665599932273229
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,float16,0,0.0271573339899381
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,1,128,1,float16,float16,0,0.014789332946141561
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,2,128,1,float16,float16,0,0.016336000214020412
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,4,128,1,float16,float16,0,0.018458666900793713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,float16,0,0.018543999642133713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,1,128,1,float16,float16,0,0.012202666451533636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,4,128,1,float16,float16,0,0.014202666779359182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,2,128,1,float16,float16,0,0.012351999680201212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,float16,0,0.014368000129858652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,float16,0,0.012426666915416718
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,float16,0,0.014463999619086584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,1,128,1,float16,float16,0,0.011749333391586939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,2,128,1,float16,float16,0,0.011727999895811081
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,4,128,1,float16,float16,0,0.012378666549921036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,float16,0,0.012410666793584824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,1,128,1,float16,float16,0,0.011551999797423681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,float16,0,0.011770666887362799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,2,128,1,float16,float16,0,0.011663999408483505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,float16,0,0.011760000139474869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,float16,0,0.011690666278203329
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,4,128,1,float16,float16,0,0.011781333635250727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,1,128,1,float16,float16,0,0.011413333316644033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,2,128,1,float16,float16,0,0.011413333316644033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,4,128,1,float16,float16,0,0.011690666278203329
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,float16,0,0.011477333803971609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,1,128,1,float16,float16,0,0.044362664222717285
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,float16,0,0.06122133135795593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,2,128,1,float16,float16,0,0.048058668772379555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,4,128,1,float16,float16,0,0.05668266614278158
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,float16,0,0.033615998923778534
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,1,128,1,float16,float16,0,0.027322667340437572
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,8,2,128,1,float16,float16,0,0.09285866220792134
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,8,4,128,1,float16,float16,0,0.12492799758911133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,8,1,128,1,float16,float16,0,0.08081066608428955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,2,128,1,float16,float16,0,0.02905600021282832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,float16,0,0.06116800010204315
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,4,128,1,float16,float16,0,0.02279466638962428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,float16,0,0.02319466571013133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,2,128,1,float16,float16,0,0.021045332153638203
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,float16,0,0.03346133232116699
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,4,128,1,float16,float16,0,0.03324799984693527
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,1,128,1,float16,float16,0,0.019424000134070713
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,float16,0,0.023056000471115112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,float16,0,0.016496000190575916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,1,128,1,float16,float16,0,0.014384000251690546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,2,128,1,float16,float16,0,0.014778666198253632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,4,128,1,float16,float16,0,0.016528000434239704
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,1,128,1,float16,float16,0,0.011850666254758835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,float16,0,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,2,128,1,float16,float16,0,0.01180800050497055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,float16,0,0.016762666404247284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,4,128,1,float16,float16,0,0.012309333930412928
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,float16,0,0.012682666381200155
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,float16,0,0.011930666863918304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,1,128,1,float16,float16,0,0.011658667276302973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,2,128,1,float16,float16,0,0.011530666301647821
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,4,128,1,float16,float16,0,0.01173866664369901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,float16,0,0.011733333269755045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,float16,0,0.011749333391586939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,1,128,1,float16,float16,0,0.011472000430027643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,2,128,1,float16,float16,0,0.011498666057984034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,4,128,1,float16,float16,0,0.011535999675591787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,float16,0,0.011477333803971609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,float16,0,0.0116799995303154
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,1,128,1,float16,float16,0,0.01129066695769628
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,2,128,1,float16,float16,0,0.011264000087976456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,4,128,1,float16,float16,0,0.011589333415031433
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,float16,0,0.011535999675591787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,0,0.01588800052801768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,0,0.033626665671666466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,0,0.04087999959786733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,0,0.026917333404223125
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,0,0.02049066623051961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,0,0.033914667864640556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,0,0.024879999458789825
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,0,0.013349333157142004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,0,0.05211733281612396
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,0,0.022287999590237934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,0,0.04065066576004028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,0,0.018245333184798557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,0,0.01293333371480306
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,0,0.019530666371186573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,0,0.022101332743962605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,0,0.026698666314284008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,0,0.024613333245118458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,0,0.013797332843144735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,0,0.012874666601419449
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,0,0.018757333358128864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,0,0.0182239996890227
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,0,0.011258666714032492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,0,0.019280000279347103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,0,0.013989333063364029
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,0,0.012448000411192576
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,0,0.018239999810854595
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,0,0.018885333091020584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,0,0.01246400053302447
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,0,0.01110400011142095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,0,0.01116266722480456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,0,0.01823466643691063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,0,0.01865600049495697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,0,0.011087999989589056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,0,0.01099733387430509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,0,0.012533333152532578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,0,0.017893332988023758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,0,0.010768000036478043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,0,0.011152000476916632
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,0,0.017898666361967724
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,0,0.012416000167528788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,0,0.012191999703645706
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,0,0.012298667182525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,0,0.01090666651725769
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,4,1,128,1,float16,float16,0,3.155242602030436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,float16,0,3.4059413274129233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,4,2,128,1,float16,float16,0,6.344693501790364
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,4,1,128,1,float16,float16,0,6.2019837697347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,float16,0,6.779290517171224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,float16,0,6.749839782714844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,4,2,128,1,float16,float16,0,3.321594556172689
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,float16,0,3.457951863606771
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,float16,0,1.952448050181071
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,4,1,128,1,float16,float16,0,1.8441759745279949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,4,2,128,1,float16,float16,0,1.8279199600219727
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,float16,0,1.9325599670410156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,4,1,128,1,float16,float16,0,12.590736389160156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,4,2,128,1,float16,float16,0,12.980581919352213
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,4,1,128,1,float16,float16,0,3.5407892862955728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,4,2,128,1,float16,float16,0,3.6804161071777344
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,float16,0,4.027423858642578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,4,1,128,1,float16,float16,0,7.187562942504883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,4,2,128,1,float16,float16,0,7.281983693440755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,float16,0,2.2557066281636557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,4,1,128,1,float16,float16,0,1.8976160685221355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,4,2,128,1,float16,float16,0,2.0128960609436035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,float16,0,2.1056639353434243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,float16,0,1.1824320157368977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,float16,0,4.018106778462728
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,4,1,128,1,float16,float16,0,1.0904373327891033
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,4,2,128,1,float16,float16,0,1.1748426755269368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,float16,0,1.1696800390879314
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,float16,0,1.5349334081013997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,4,1,128,1,float16,float16,0,2.491637388865153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,float16,0,2.885525385538737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,4,2,128,1,float16,float16,0,2.630095958709717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,float16,0,2.8989334106445312
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,4,1,128,1,float16,float16,0,4.927018801371257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,4,1,128,1,float16,float16,0,1.326581319173177
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,4,2,128,1,float16,float16,0,5.139861424763997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,4,2,128,1,float16,float16,0,1.4183573722839355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,float16,0,0.8473813533782959
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,4,1,128,1,float16,float16,0,0.7751306692759196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,float16,0,1.521440029144287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,float16,0,0.8445706367492676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,4,2,128,1,float16,float16,0,0.8330346743265787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,float16,0,1.9922986030578613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,4,1,128,1,float16,float16,0,3.1545279820760093
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,4,2,128,1,float16,float16,0,3.319194793701172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,4,1,128,1,float16,float16,0,1.6280585924784343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,float16,0,3.7454185485839844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,float16,0,3.7535680135091147
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,float16,0,1.0529066721598308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,4,2,128,1,float16,float16,0,1.7461439768473308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,4,2,128,1,float16,float16,0,0.9541013240814209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,4,1,128,1,float16,float16,0,0.8980213006337484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,float16,0,1.992634614308675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,4,1,128,1,float16,float16,0,6.238815943400065
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,float16,0,0.6007093191146851
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,float16,0,1.0559679667154949
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,4,1,128,1,float16,float16,0,0.5285280148188273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,4,2,128,1,float16,float16,0,0.5490986506144205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,4,2,128,1,float16,float16,0,6.696922938028972
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,float16,0,0.5933279991149902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,4,1,128,1,float16,float16,0,0.9619680245717367
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,float16,0,1.2875253359476726
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,4,1,128,1,float16,float16,0,1.839157263437907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,4,2,128,1,float16,float16,0,1.9855146408081055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,float16,0,2.2922399838765464
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,float16,0,2.2979466120402017
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,4,2,128,1,float16,float16,0,1.043562650680542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,float16,0,0.6667893727620443
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,4,1,128,1,float16,float16,0,0.5517760117848715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,float16,0,1.2307733694712322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,4,1,128,1,float16,float16,0,3.6299308141072593
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,4,2,128,1,float16,float16,0,0.5905280113220215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,float16,0,0.6621386607487997
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,4,2,128,1,float16,float16,0,3.8821067810058594
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,float16,0,0.3963199853897095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,4,1,128,1,float16,float16,0,0.33590400218963623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,4,2,128,1,float16,float16,0,0.3516319990158081
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,float16,0,0.40383466084798175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,4,1,128,1,float16,float16,0,0.8821866512298584
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,float16,0,1.1956213315327961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,4,2,128,1,float16,float16,0,1.8606613477071126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,4,1,128,1,float16,float16,0,1.7332480748494465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,float16,0,2.2940853436787925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,float16,0,2.3034399350484214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,4,2,128,1,float16,float16,0,0.9924426873524984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,4,2,128,1,float16,float16,0,0.5336320002873739
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,float16,0,1.199903964996338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,float16,0,0.6387840112050375
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,4,1,128,1,float16,float16,0,0.4931786855061849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,4,2,128,1,float16,float16,0,3.669077237447103
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,4,1,128,1,float16,float16,0,3.3688319524129233
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,float16,0,0.6332799990971884
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,float16,0,0.3646879990895589
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,4,1,128,1,float16,float16,0,0.28245333830515545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,4,2,128,1,float16,float16,0,0.31330132484436035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,float16,0,0.20828799406687418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,4,1,128,1,float16,float16,0,0.18601065874099731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,float16,0,0.20951465765635172
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,float16,0,0.3622186581293742
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,4,2,128,1,float16,float16,0,0.19826666514078775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,float16,0,0.7757226626078287
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,float16,0,1.4655839602152507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,4,2,128,1,float16,float16,0,1.1871840159098308
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,4,1,128,1,float16,float16,0,1.021392027537028
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,float16,0,1.4729706446329753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,4,1,128,1,float16,float16,0,0.5479466517766317
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,4,2,128,1,float16,float16,0,2.2008426984151206
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,4,2,128,1,float16,float16,0,0.6162613232930502
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,float16,0,0.42604267597198486
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,4,1,128,1,float16,float16,0,1.9366079966227214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,float16,0,0.7653066317240397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,4,2,128,1,float16,float16,0,0.34086398283640545
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,4,1,128,1,float16,float16,0,0.3072693347930908
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,4,2,128,1,float16,float16,0,0.1986400087674459
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,float16,0,0.23932266235351562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,float16,0,0.4211626847585042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,4,1,128,1,float16,float16,0,0.16961065928141275
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,float16,0,0.13370666901270548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,4,1,128,1,float16,float16,0,0.11727999647458394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,4,2,128,1,float16,float16,0,0.12331199645996094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,float16,0,0.1339040001233419
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,float16,0,0.2360853354136149
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,float16,0,0.8087519804636637
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,float16,0,1.5791093508402507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,4,1,128,1,float16,float16,0,0.9658613204956055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,float16,0,1.5729279518127441
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,4,2,128,1,float16,float16,0,1.1504213015238445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,4,1,128,1,float16,float16,0,0.5050826470057169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,4,2,128,1,float16,float16,0,0.6075946489969889
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,float16,0,0.43298665682474774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,4,1,128,1,float16,float16,0,1.8882400194803874
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,float16,0,0.8126506805419922
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,4,2,128,1,float16,float16,0,0.3304693301518758
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,4,1,128,1,float16,float16,0,0.28035734097162884
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,float16,0,0.2406346599260966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,4,2,128,1,float16,float16,0,2.2122933069864907
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,4,2,128,1,float16,float16,0,0.18659732739130655
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,float16,0,0.43010131518046063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,float16,0,0.13269866506258646
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,float16,0,0.23850667476654053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,4,1,128,1,float16,float16,0,0.15753066539764404
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,4,1,128,1,float16,float16,0,0.10236799716949463
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,float16,0,0.08935999870300293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,float16,0,0.1301759978135427
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,4,1,128,1,float16,float16,0,0.07748266557852428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,4,2,128,1,float16,float16,0,0.0807360013326009
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,float16,0,0.08949333429336548
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,4,2,128,1,float16,float16,0,0.10621333122253418
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,float16,0,1.0573440392812092
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,4,1,128,1,float16,float16,0,0.5980853239695231
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,float16,0,1.0572213331858318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,float16,0,0.5493706862131754
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,4,2,128,1,float16,float16,0,0.7426880200703939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,4,1,128,1,float16,float16,0,0.31701866785685223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,4,2,128,1,float16,float16,0,0.3956000010172526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,float16,0,0.29543999830881756
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,float16,0,0.551423986752828
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,4,1,128,1,float16,float16,0,1.156335989634196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,4,2,128,1,float16,float16,0,0.21581866343816122
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,4,1,128,1,float16,float16,0,0.18024533987045288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,4,1,128,1,float16,float16,0,0.10604266325632732
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,float16,0,0.1706613302230835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,4,2,128,1,float16,float16,0,1.4122613271077473
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,float16,0,0.29365867376327515
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,float16,0,0.08127466837565105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,4,1,128,1,float16,float16,0,0.061146666606267296
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,4,2,128,1,float16,float16,0,0.06754666566848755
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,4,2,128,1,float16,float16,0,0.12411200006802876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,float16,0,0.05672533313433329
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,float16,0,0.16697067022323608
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,4,2,128,1,float16,float16,0,0.050010666251182556
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,float16,0,0.05682133138179779
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,4,1,128,1,float16,float16,0,0.047685335079828896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,float16,0,0.0794293334086736
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,4,1,128,1,float16,float16,0,0.6103786627451578
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,4,1,128,1,float16,float16,0,0.3184799949328105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,float16,0,0.6307413180669149
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,4,2,128,1,float16,float16,0,0.7953706582387289
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,4,2,128,1,float16,float16,0,0.4294133186340332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,float16,0,0.33019200960795086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,float16,0,0.6255786816279093
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,4,1,128,1,float16,float16,0,0.17290133237838745
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,float16,0,1.2199626763661702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,4,1,128,1,float16,float16,0,1.1808799902598064
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,float16,0,0.3296213348706563
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,4,2,128,1,float16,float16,0,0.22698666652043661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,float16,0,1.2234613100687664
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,4,1,128,1,float16,float16,0,0.09731200337409973
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,float16,0,0.18054399887720743
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,float16,0,0.08835200468699138
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,4,1,128,1,float16,float16,0,0.058917333682378135
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,float16,0,0.08740267157554626
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,4,2,128,1,float16,float16,0,1.5258612632751465
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,4,2,128,1,float16,float16,0,0.04956800242265066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,float16,0,0.057962665955225624
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,4,1,128,1,float16,float16,0,0.04576000074545542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,float16,0,0.05824000140031179
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,4,1,128,1,float16,float16,0,0.0349440003434817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,float16,0,0.04005866746107737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,4,2,128,1,float16,float16,0,0.03638399889071783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,4,2,128,1,float16,float16,0,0.12821333607037863
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,float16,0,0.1805760065714518
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,4,2,128,1,float16,float16,0,0.0673173318306605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,4,1,128,1,float16,float16,0,0.4357440074284871
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,4,1,128,1,float16,float16,0,0.22923199335734049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,float16,0,0.04001600046952566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,float16,0,0.5368586778640747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,4,2,128,1,float16,float16,0,0.6175520022710165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,4,1,128,1,float16,float16,0,0.8474559783935547
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,float16,0,1.050645351409912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,4,2,128,1,float16,float16,0,0.32469866673151654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,float16,0,1.0508586565653484
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,float16,0,0.2801919976870219
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,float16,0,0.5354613463083903
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,4,2,128,1,float16,float16,0,1.2024426460266113
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,4,1,128,1,float16,float16,0,0.12417599558830261
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,float16,0,0.15015467007954916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,4,1,128,1,float16,float16,0,0.06941866874694824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,float16,0,0.1508799990018209
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,4,1,128,1,float16,float16,0,0.04299733539422353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,4,2,128,1,float16,float16,0,0.050901333491007485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,float16,0,0.07233066856861115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,float16,0,0.04201599955558777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,4,1,128,1,float16,float16,0,0.02977599948644638
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,4,2,128,1,float16,float16,0,0.1794453263282776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,4,2,128,1,float16,float16,0,0.09566400448481242
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,float16,0,0.27832533915837604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,4,2,128,1,float16,float16,0,0.033471999069054924
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,float16,0,0.042090664307276406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,float16,0,0.028346667687098186
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,4,2,128,1,float16,float16,0,0.024682665864626568
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,float16,0,0.024693332612514496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,4,1,128,1,float16,float16,0,0.021136000752449036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,4,1,128,1,float16,float16,0,0.02327466756105423
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,4,2,128,1,float16,float16,0,0.022800001005331676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,float16,0,0.02460266649723053
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,float16,0,0.028624000648657482
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,float16,0,0.07195733487606049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,4,1,128,1,float16,float16,0,0.18133334318796793
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,4,2,128,1,float16,float16,0,0.2866133252779643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,float16,0,0.49297066529591876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,float16,0,0.25515200694402057
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,4,1,128,1,float16,float16,0,0.09760000308354695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,float16,0,0.487829327583313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,4,2,128,1,float16,float16,0,0.14958932995796204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,4,1,128,1,float16,float16,0,0.349455992380778
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,4,2,128,1,float16,float16,0,0.08227733274300893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,4,1,128,1,float16,float16,0,0.052426666021347046
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,float16,0,0.05945600072542826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,4,2,128,1,float16,float16,0,0.5253706773122152
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,float16,0,0.13525866468747458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,float16,0,0.06057066718737284
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,4,2,128,1,float16,float16,0,0.03965866565704346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,float16,0,0.13581333557764688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,float16,0,0.03619733452796936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,float16,0,0.25199466943740845
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,float16,0,0.02311466634273529
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,4,1,128,1,float16,float16,0,0.03128000100453695
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,float16,0,0.03621333340803782
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,4,1,128,1,float16,float16,0,0.023733332753181458
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,4,2,128,1,float16,float16,0,0.019178666174411774
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,4,1,128,1,float16,float16,0,0.01756799966096878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,float16,0,0.019226666539907455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,4,2,128,1,float16,float16,0,0.017237332959969837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,4,1,128,1,float16,float16,0,0.015754666179418564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,float16,0,0.01923199991385142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,4,1,128,1,float16,float16,0,0.015216000378131866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,float16,0,0.01738133281469345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,float16,0,0.02309866746266683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,4,2,128,1,float16,float16,0,0.01578666642308235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,float16,0,0.017258666455745697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,4,2,128,1,float16,float16,0,0.02749866743882497
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,float16,0,0.12990933656692505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,4,1,128,1,float16,float16,0,0.04275199770927429
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,4,1,128,1,float16,float16,0,0.08362133304278056
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,4,2,128,1,float16,float16,0,0.13954666256904602
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,float16,0,0.24328533808390299
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,float16,0,0.2437546650568644
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,4,2,128,1,float16,float16,0,0.07502399881680806
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,float16,0,0.05489600201447805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,float16,0,0.12965333461761475
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,4,1,128,1,float16,float16,0,0.027306665976842243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,4,2,128,1,float16,float16,0,0.03549333413441976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,float16,0,0.031018666923046112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,4,1,128,1,float16,float16,0,0.16276266177495322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,4,2,128,1,float16,float16,0,0.26796799898147583
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,float16,0,0.031082667410373688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,float16,0,0.05513066550095876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,float16,0,0.019909333437681198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,4,1,128,1,float16,float16,0,0.018922666708628338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,4,2,128,1,float16,float16,0,0.022469334304332733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,4,1,128,1,float16,float16,0,0.014384000251690546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,4,2,128,1,float16,float16,0,0.016074666132529575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,4,1,128,1,float16,float16,0,0.012485332787036896
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,4,1,128,1,float16,float16,0,0.012351999680201212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,float16,0,0.015957333147525787
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,float16,0,0.019808000574509304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,float16,0,0.014240000396966934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,float16,0,0.015882667154073715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,4,2,128,1,float16,float16,0,0.01239466667175293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,4,2,128,1,float16,float16,0,0.014032000054915747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,float16,0,0.01402666668097178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,float16,0,0.012335999558369318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,4,1,128,1,float16,float16,0,0.011850666254758835
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,float16,0,0.012437333663304647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,4,2,128,1,float16,float16,0,0.011877333124478659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,float16,0,0.0545066644748052
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,4,1,128,1,float16,float16,0,0.026485333840052288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,4,1,128,1,float16,float16,0,0.043040002385775246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,4,2,128,1,float16,float16,0,0.07392000158627827
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,float16,0,0.12990400195121765
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,float16,0,0.1296266714731852
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,4,2,128,1,float16,float16,0,0.03485333422819773
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,float16,0,0.030645333230495453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,float16,0,0.056277334690093994
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,4,1,128,1,float16,float16,0,0.018954666952292126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,4,1,128,1,float16,float16,0,0.08323733508586884
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,4,2,128,1,float16,float16,0,0.022298666338125866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,float16,0,0.01947733387351036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,4,2,128,1,float16,float16,0,0.1381386617819468
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,float16,0,0.030773334205150604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,4,1,128,1,float16,float16,0,0.014111999422311783
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,4,2,128,1,float16,float16,0,0.015893333901961643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,4,1,128,1,float16,float16,0,0.012517333030700684
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,float16,0,0.015967999895413715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,4,2,128,1,float16,float16,0,0.013978666315476099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,float16,0,0.019573333362738293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,float16,0,0.013893333574136099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,4,1,128,1,float16,float16,0,0.012181332955757776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,float16,0,0.01590399940808614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,4,2,128,1,float16,float16,0,0.01231466606259346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,float16,0,0.012362666428089142
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,float16,0,0.013994666437307993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,4,1,128,1,float16,float16,0,0.011749333391586939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,float16,0,0.012298667182525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,float16,0,0.011887999872366587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,4,2,128,1,float16,float16,0,0.011866666376590729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,4,1,128,1,float16,float16,0,0.011749333391586939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,float16,0,0.011802667131026586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,4,2,128,1,float16,float16,0,0.011760000139474869
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,float16,0,0.0345920001467069
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,float16,0,0.06178666651248932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,4,2,128,1,float16,float16,0,0.040448000033696495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,4,1,128,1,float16,float16,0,0.03172266731659571
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,float16,0,0.06258133550484975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,4,1,128,1,float16,float16,0,0.02229333420594533
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,float16,0,0.021615999440352123
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,4,1,128,1,float16,float16,0,0.016261332978804905
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,float16,0,0.03450666616360346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,4,2,128,1,float16,float16,0,0.025994665920734406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,4,1,128,1,float16,float16,0,0.0565226674079895
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,4,2,128,1,float16,float16,0,0.017946666727463405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,4,2,128,1,float16,float16,0,0.08802133798599243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,float16,0,0.02184533327817917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,4,1,128,1,float16,float16,0,0.012245333443085352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,4,2,128,1,float16,float16,0,0.013834666460752487
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,float16,0,0.01587733378012975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,float16,0,0.013877333452304205
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,float16,0,0.01597333326935768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,4,1,128,1,float16,float16,0,0.011898666620254517
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,4,2,128,1,float16,float16,0,0.012293333808581034
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,float16,0,0.012250666817029318
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,float16,0,0.01392000044385592
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,4,1,128,1,float16,float16,0,0.011663999408483505
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,4,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,float16,0,0.012389333297808966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,4,1,128,1,float16,float16,0,0.011461333682139715
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,4,2,128,1,float16,float16,0,0.011488000551859537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,float16,0,0.011551999797423681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,float16,0,0.011642667154471079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,4,1,128,1,float16,float16,0,0.011424000064531961
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,float16,0,0.011685332904259363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,4,2,128,1,float16,float16,0,0.01181866725285848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,float16,0,0.026421333352724712
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,float16,0,0.04101333270470301
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,4,1,128,1,float16,float16,0,0.028933333853880566
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,4,2,128,1,float16,float16,0,0.03257599969704946
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,float16,0,0.04115733255942663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,4,1,128,1,float16,float16,0,0.020725333442290623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,4,1,128,1,float16,float16,0,0.014885333677132925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,4,2,128,1,float16,float16,0,0.022410665949185688
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,float16,0,0.018378666291634243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,4,2,128,1,float16,float16,0,0.0162773331006368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,float16,0,0.02641066660483678
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,4,2,128,1,float16,float16,0,0.05574400226275126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,4,1,128,1,float16,float16,0,0.047050664822260536
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,float16,0,0.017994667092959087
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,float16,0,0.014096000542243322
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,4,1,128,1,float16,float16,0,0.011866666376590729
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,float16,0,0.01240533341964086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,4,2,128,1,float16,float16,0,0.012240000069141388
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,float16,0,0.014261333892742792
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,4,1,128,1,float16,float16,0,0.011770666887362799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,4,2,128,1,float16,float16,0,0.011749333391586939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,float16,0,0.011706666400035223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,float16,0,0.012213333199421564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,4,1,128,1,float16,float16,0,0.011440000186363855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,4,2,128,1,float16,float16,0,0.011584000041087469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,float16,0,0.011472000430027643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,4,1,128,1,float16,float16,0,0.011296000331640244
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,4,2,128,1,float16,float16,0,0.011333333949247995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,float16,0,0.011749333391586939
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,float16,0,0.011322667201360067
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,float16,0,0.011610666910807291
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,4,1,128,1,float16,float16,0,0.011194666226704916
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,4,2,128,1,float16,float16,0,0.011450666934251785
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,float16,0,0.011584000041087469
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,0,0.01823466643691063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,0,0.013349333157142004
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,0,0.02496533344189326
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,0,0.01573333392540614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,0,0.02510933329661687
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,0,0.026554666459560394
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,0,0.021802666286627453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,0,0.018218666315078735
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,0,0.013946666071812311
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,0,0.012938667088747025
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,0,0.018725333114465077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,0,0.020207999895016353
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,0,0.011477333803971609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,0,0.012773333738247553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,0,0.033701332906881966
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,0,0.014090667168299357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,0,0.01823466643691063
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,0,0.011301333705584208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,0,0.011226666470368704
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,0,0.012560000022252401
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,0,0.018112000077962875
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,0,0.01097600037852923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,0,0.011039999624093374
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,0,0.01231466606259346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,0,0.01830400029818217
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,0,0.011007999380429586
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,0,0.01108266661564509
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,0,0.012175999581813812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,0,0.017797333498795826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,0,0.010938666760921478
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,0,0.010661333799362183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,0,0.012138667205969492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,0,0.012543999900420507
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,0,0.011136000355084738
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,float16,0,1.84550937016805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,2,1,128,1,float16,float16,0,1.758911927541097
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,float16,0,1.875439961751302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,2,1,128,1,float16,float16,0,3.202853202819824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,float16,0,3.3811572392781577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,float16,0,3.381082534790039
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,2,1,128,1,float16,float16,0,1.0195679664611816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,float16,0,1.0695520242055256
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,float16,0,1.0676853656768799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,float16,0,1.1560479799906414
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,2,1,128,1,float16,float16,0,1.072208007176717
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,2,1,128,1,float16,float16,0,1.8752800623575847
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,float16,0,2.0385173161824546
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,2,1,128,1,float16,float16,0,6.321642557779948
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,float16,0,2.041600068410238
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,float16,0,0.7053226629892985
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,float16,0,1.1394826571146648
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,2,1,128,1,float16,float16,0,0.6537546714146932
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,float16,0,0.7126080195109049
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,2,1,128,1,float16,float16,0,3.5632479985555015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,float16,0,1.4975786209106445
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,2,1,128,1,float16,float16,0,0.7577706972757975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,float16,0,0.8332213560740153
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,float16,0,0.854528029759725
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,2,1,128,1,float16,float16,0,1.3709333737691243
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,float16,0,1.4880480766296387
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,2,1,128,1,float16,float16,0,2.6559839248657227
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,float16,0,0.5252533356348673
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,2,1,128,1,float16,float16,0,0.4904746611913045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,float16,0,0.5258080164591471
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,float16,0,1.0393706957499187
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,2,1,128,1,float16,float16,0,0.9392106533050537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,float16,0,1.8718026479085286
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,2,1,128,1,float16,float16,0,1.6909279823303223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,float16,0,1.0277653535207112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,float16,0,1.9215466181437175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,float16,0,0.587664008140564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,2,1,128,1,float16,float16,0,0.5301760037740072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,2,1,128,1,float16,float16,0,0.35172800223032635
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,float16,0,0.37671999136606854
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,float16,0,0.5840746561686198
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,2,1,128,1,float16,float16,0,3.2848691940307617
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,float16,0,0.37987732887268066
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,float16,0,0.6584746837615967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,2,1,128,1,float16,float16,0,0.5920960108439127
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,float16,0,0.6528160174687704
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,2,1,128,1,float16,float16,0,1.0187573432922363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,float16,0,1.1632800102233887
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,float16,0,1.1680320103963215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,float16,0,0.3925066788991292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,2,1,128,1,float16,float16,0,0.3455359935760498
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,float16,0,0.24215465784072876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,2,1,128,1,float16,float16,0,1.9117546081542969
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,float16,0,0.39868799845377606
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,2,1,128,1,float16,float16,0,0.2271946668624878
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,float16,0,0.24054932594299316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,float16,0,0.3561226526896159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,float16,0,0.626362681388855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,float16,0,0.6173973480860392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,2,1,128,1,float16,float16,0,0.5231200059254965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,float16,0,1.1376852989196777
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,float16,0,1.1475573380788167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,2,1,128,1,float16,float16,0,0.30428266525268555
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,2,1,128,1,float16,float16,0,0.9416639804840088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,float16,0,0.19979733228683472
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,2,1,128,1,float16,float16,0,0.184879998366038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,float16,0,0.35596267382303876
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,float16,0,0.13808000087738037
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,2,1,128,1,float16,float16,0,0.126991997162501
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,float16,0,0.21559999386469522
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,2,1,128,1,float16,float16,0,1.7913653055826824
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,float16,0,0.13524267077445984
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,float16,0,0.40591998895009357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,float16,0,0.7486080328623453
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,float16,0,0.740506649017334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,2,1,128,1,float16,float16,0,0.5875786542892456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,float16,0,0.40488000710805255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,2,1,128,1,float16,float16,0,0.326693336168925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,float16,0,0.2316640019416809
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,2,1,128,1,float16,float16,0,0.1863200068473816
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,float16,0,0.2318026622136434
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,2,1,128,1,float16,float16,0,1.0958399772644043
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,float16,0,0.132341335217158
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,2,1,128,1,float16,float16,0,0.11974933743476868
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,float16,0,0.09029866258303325
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,2,1,128,1,float16,float16,0,0.08517332871754964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,float16,0,0.1316266655921936
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,float16,0,0.09044800202051799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,2,1,128,1,float16,float16,0,0.30963732798894245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,float16,0,0.4095146656036377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,float16,0,0.41703999042510986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,float16,0,0.2381920019785563
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,2,1,128,1,float16,float16,0,0.5717600186665853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,float16,0,0.7696959972381592
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,float16,0,0.7444533507029215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,2,1,128,1,float16,float16,0,1.084106683731079
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,float16,0,0.13110933701197305
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,2,1,128,1,float16,float16,0,0.17659733692804971
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,float16,0,0.23136534293492636
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,2,1,128,1,float16,float16,0,0.10872000455856323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,float16,0,0.08891733487447102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,float16,0,0.06284800171852112
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,float16,0,0.06272533535957336
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,2,1,128,1,float16,float16,0,0.05941333373387655
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,float16,0,0.12989866733551025
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,float16,0,0.08886933326721191
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,2,1,128,1,float16,float16,0,0.3670933246612549
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,2,1,128,1,float16,float16,0,0.2023520072301229
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,2,1,128,1,float16,float16,0,0.0805866668621699
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,float16,0,0.28276266654332477
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,float16,0,0.5099733273188273
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,float16,0,0.16061866283416748
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,float16,0,0.2866186698277791
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,float16,0,0.5006399949391683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,2,1,128,1,float16,float16,0,0.11427199840545654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,float16,0,0.16355733076731363
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,2,1,128,1,float16,float16,0,0.6894026597340902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,float16,0,0.0779306689898173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,2,1,128,1,float16,float16,0,0.06523199876149495
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,float16,0,0.05606933434804281
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,float16,0,0.04906666775544485
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,float16,0,0.04902400076389313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,float16,0,0.055973331133524575
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,float16,0,0.07830399771531422
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,2,1,128,1,float16,float16,0,0.0496319979429245
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,2,1,128,1,float16,float16,0,0.20761066675186157
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,2,1,128,1,float16,float16,0,0.04637866715590159
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,float16,0,0.3118666609128316
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,2,1,128,1,float16,float16,0,0.3856213490168254
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,float16,0,0.17377599080403647
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,2,1,128,1,float16,float16,0,0.11717866857846577
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,float16,0,0.5682186683019003
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,float16,0,0.30799466371536255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,float16,0,0.17199466625849405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,2,1,128,1,float16,float16,0,0.06543999910354614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,float16,0,0.0865280032157898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,float16,0,0.5590560038884481
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,float16,0,0.08763200044631958
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,float16,0,0.03980266551176707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,2,1,128,1,float16,float16,0,0.036159999668598175
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,float16,0,0.057162667314211525
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,2,1,128,1,float16,float16,0,0.7315786679585775
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,float16,0,0.03565333286921183
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,2,1,128,1,float16,float16,0,0.04860800007979075
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,2,1,128,1,float16,float16,0,0.03421333432197571
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,float16,0,0.05678399900595347
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,float16,0,0.0359199990828832
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,float16,0,0.03961066653331121
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,2,1,128,1,float16,float16,0,0.15813866257667542
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,float16,0,0.1418506701787313
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,float16,0,0.2629813353220622
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,float16,0,0.4687946637471517
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,float16,0,0.2596319913864136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,2,1,128,1,float16,float16,0,0.08835732936859131
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,float16,0,0.14102400342623392
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,float16,0,0.06748799979686737
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,2,1,128,1,float16,float16,0,0.29206933577855426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,float16,0,0.06929066777229309
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,float16,0,0.041050667564074196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,float16,0,0.4705919822057088
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,float16,0,0.04098666707674662
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,2,1,128,1,float16,float16,0,0.03283733377854029
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,2,1,128,1,float16,float16,0,0.04842666784922282
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,float16,0,0.027989332874615986
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,2,1,128,1,float16,float16,0,0.024586667617162068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,2,1,128,1,float16,float16,0,0.0225600004196167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,float16,0,0.027834666272004444
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,float16,0,0.024245334168275196
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,2,1,128,1,float16,float16,0,0.021061333517233532
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,float16,0,0.022570667167504627
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,float16,0,0.022778667509555817
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,float16,0,0.02426133304834366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,2,1,128,1,float16,float16,0,0.5588373343149821
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,float16,0,0.12900267044703165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,2,1,128,1,float16,float16,0,0.07273600002129872
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,2,1,128,1,float16,float16,0,0.13212800025939941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,float16,0,0.2281493345896403
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,float16,0,0.12668800354003906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,float16,0,0.05762133498986562
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,2,1,128,1,float16,float16,0,0.036533333361148834
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,float16,0,0.23735467592875162
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,float16,0,0.058176000912984215
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,float16,0,0.02232533444960912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,float16,0,0.035216001172860466
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,2,1,128,1,float16,float16,0,0.026874666412671406
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,float16,0,0.01863466699918111
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,float16,0,0.02239466706911723
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,float16,0,0.03484266748030981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,2,1,128,1,float16,float16,0,0.24758932987848917
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,2,1,128,1,float16,float16,0,0.019039999693632126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,float16,0,0.018735999862353008
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,2,1,128,1,float16,float16,0,0.016879999389251072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,float16,0,0.016837333639462788
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,2,1,128,1,float16,float16,0,0.015599999576807022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,float16,0,0.01685333376129468
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,float16,0,0.01526933287580808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,2,1,128,1,float16,float16,0,0.015146666516860327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,float16,0,0.015306666493415833
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,float16,0,0.05272000034650167
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,2,1,128,1,float16,float16,0,0.032629333436489105
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,2,1,128,1,float16,float16,0,0.06285333136717479
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,float16,0,0.12249599893887837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,float16,0,0.05126399795214335
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,float16,0,0.1223360002040863
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,2,1,128,1,float16,float16,0,0.021685334543387096
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,float16,0,0.03010133405526479
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,float16,0,0.01929066702723503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,2,1,128,1,float16,float16,0,0.01597333326935768
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,2,1,128,1,float16,float16,0,0.01402666668097178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,float16,0,0.01562133307258288
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,float16,0,0.029738667110602062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,float16,0,0.01947733387351036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,float16,0,0.015669333438078564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,2,1,128,1,float16,float16,0,0.12222400307655334
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,2,1,128,1,float16,float16,0,0.01239466667175293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,float16,0,0.013733333597580591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,float16,0,0.012053333222866058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,2,1,128,1,float16,float16,0,0.011882666498422623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,float16,0,0.0122079998254776
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,float16,0,0.011973333855470022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,2,1,128,1,float16,float16,0,0.011973333855470022
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,float16,0,0.011834666132926941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,float16,0,0.029711998999118805
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,2,1,128,1,float16,float16,0,0.0216799999276797
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,float16,0,0.02959999938805898
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,2,1,128,1,float16,float16,0,0.031957333286603294
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,float16,0,0.05171200136343638
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,float16,0,0.055205335219701133
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,2,1,128,1,float16,float16,0,0.015696000307798386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,float16,0,0.019199999670187633
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,float16,0,0.019189332922299702
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,float16,0,0.015498666713635126
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,2,1,128,1,float16,float16,0,0.013605333864688873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,float16,0,0.015439999600251516
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,2,1,128,1,float16,float16,0,0.012400000045696894
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,float16,0,0.012170666207869848
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,float16,0,0.013914667069911957
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,2,1,128,1,float16,float16,0,0.011616000284751257
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,float16,0,0.012042666474978128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,float16,0,0.011733333269755045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,2,1,128,1,float16,float16,0,0.01173866664369901
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,float16,0,0.011733333269755045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,float16,0,0.01166933278242747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,2,1,128,1,float16,float16,0,0.011786667009194693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,float16,0,0.011706666400035223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,2,1,128,1,float16,float16,0,0.06465599934260051
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,float16,0,0.015466666469971338
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,float16,0,0.021162666380405426
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,float16,0,0.02107733239730199
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,2,1,128,1,float16,float16,0,0.017680000513792038
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,float16,0,0.03345066557327906
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,2,1,128,1,float16,float16,0,0.025285333395004272
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,float16,0,0.033530667424201965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,float16,0,0.013733333597580591
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,2,1,128,1,float16,float16,0,0.013605333864688873
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,float16,0,0.01570133368174235
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,2,1,128,1,float16,float16,0,0.012351999680201212
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,float16,0,0.01221866657336553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,float16,0,0.013701333353916803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,2,1,128,1,float16,float16,0,0.011744000017642975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,float16,0,0.012128000458081564
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,float16,0,0.011706666400035223
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,2,1,128,1,float16,float16,0,0.011733333269755045
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,float16,0,0.011621333658695221
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,float16,0,0.011557333171367645
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,2,1,128,1,float16,float16,0,0.011317333827416102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,float16,0,0.011429333438475927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,float16,0,0.011317333827416102
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,2,1,128,1,float16,float16,0,0.011365332951148352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,float16,0,0.011445333560307821
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,2,1,128,1,float16,float16,0,0.03728000074625015
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,float16,0,0.017952000101407368
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,2,1,128,1,float16,float16,0,0.01621333385507266
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,float16,0,0.017738666385412216
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,float16,0,0.02587733417749405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,2,1,128,1,float16,float16,0,0.022181332111358643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,float16,0,0.025573333104451496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,float16,0,0.013946666071812311
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,2,1,128,1,float16,float16,0,0.01227733368674914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,float16,0,0.013770667215188345
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,float16,0,0.012047999848922094
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,float16,0,0.01209066684047381
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,2,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,float16,0,0.012053333222866058
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,2,1,128,1,float16,float16,0,0.011871999750534693
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,float16,0,0.011482667177915573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,float16,0,0.01166933278242747
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,2,1,128,1,float16,float16,0,0.01126933346192042
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,float16,0,0.011407999942700068
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,float16,0,0.011429333438475927
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,2,1,128,1,float16,float16,0,0.011440000186363855
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,2,1,128,1,float16,float16,0,0.03160533308982849
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,float16,0,0.011359999577204386
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,float16,0,0.01139733319481214
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,2,1,128,1,float16,float16,0,0.011338666081428528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,float16,0,0.011338666081428528
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,0,0.01292266696691513
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,0,0.013701333353916803
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,0,0.014080000420411428
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,0,0.01825599993268649
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,0,0.015674666812022526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,0,0.018330667167901993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,0,0.011258666714032492
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,0,0.012944000462690989
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,0,0.011381333072980246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,0,0.011333333949247995
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,0,0.012815999488035837
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,0,0.010784000158309937
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,0,0.01090666651725769
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,0,0.01240533341964086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,0,0.011125333607196808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,0,0.010911999891201654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,0,0.012495999534924826
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,0,0.010751999914646149
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,0,0.010784000158309937
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,0,0.012261333564917246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,0,0.010709332923094431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,0,0.0201706662774086
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,0,0.01089599976936976
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,0,0.010629333555698395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,0,0.012304000556468964
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,float16,0,0.6924906571706136
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,float16,0,0.6975680192311605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,float16,0,1.0493120352427165
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,float16,0,1.0447893142700195
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,float16,0,1.1219360033671062
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,float16,0,1.806096076965332
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,float16,0,1.8196320533752441
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,float16,0,0.3746986786524455
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,float16,0,0.6922453244527181
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,float16,0,0.6890986760457357
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,float16,0,0.3758399883906047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,float16,0,1.1105493704477947
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,float16,0,0.8105546633402506
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,float16,0,0.30081599950790405
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,float16,0,0.803167978922526
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,float16,0,0.3002079923947652
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,float16,0,0.5135840177536011
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,float16,0,0.5192106564839681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,float16,0,0.22549867630004883
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,float16,0,0.9917439619700114
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,float16,0,0.3712693452835083
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,float16,0,0.36765865484873456
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,float16,0,0.5658613443374634
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,float16,0,0.5657386779785156
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,float16,0,0.22552533944447836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,float16,0,0.976149320602417
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,float16,0,0.3765706618626912
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,float16,0,0.6249973376592001
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,float16,0,0.3744693199793498
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,float16,0,0.23697600762049356
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,float16,0,0.23648534218470255
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,float16,0,0.15800000230471292
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,float16,0,0.1579200029373169
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,float16,0,0.6223946809768677
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,float16,0,0.1325973371664683
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,float16,0,0.13272000352541605
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,float16,0,0.20217599471410116
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,float16,0,0.20122132698694864
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,float16,0,0.34020264943440753
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,float16,0,0.33826665083567303
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,float16,0,0.10846400260925293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,float16,0,0.1086133321126302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,float16,0,0.5784960190455118
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,float16,0,0.5696746508280436
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,float16,0,0.12917332847913107
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,float16,0,0.12821867068608603
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,float16,0,0.2126986583073934
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,float16,0,0.2124533255894979
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,float16,0,0.37379733721415204
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,float16,0,0.3715999921162923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,float16,0,0.08331199983755748
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,float16,0,0.08839999636014302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,float16,0,0.08358400066693623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,float16,0,0.08822400371233623
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,float16,0,0.21176532904307047
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,float16,0,0.12361600001653035
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,float16,0,0.21079466740290323
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,float16,0,0.08641067147254944
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,float16,0,0.1251413325468699
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,float16,0,0.37135998407999676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,float16,0,0.37371734778086346
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,float16,0,0.06203199923038483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,float16,0,0.05865600208441416
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,float16,0,0.08589333295822144
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,float16,0,0.058746665716171265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,float16,0,0.061893333991368614
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,float16,0,0.14538666605949402
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,float16,0,0.053770666321118675
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,float16,0,0.24970666567484537
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,float16,0,0.07338666419188182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,float16,0,0.07409066458543141
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,float16,0,0.251802663008372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,float16,0,0.1441439986228943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,float16,0,0.048565333088239036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,float16,0,0.04635733366012573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,float16,0,0.05403199791908264
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,float16,0,0.04619733492533366
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,float16,0,0.04862933357556661
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,float16,0,0.08156799773375194
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,float16,0,0.0809440016746521
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,float16,0,0.053904001911481224
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,float16,0,0.1511146624883016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,float16,0,0.2701386610666911
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,float16,0,0.15134933590888977
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,float16,0,0.27052799860636395
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,float16,0,0.053930665055910744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,float16,0,0.039146666725476585
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,float16,0,0.038773333032925926
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,float16,0,0.03563733398914337
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,float16,0,0.03404266635576884
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,float16,0,0.03563733398914337
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,float16,0,0.03393599887688955
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,float16,0,0.06371733546257019
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,float16,0,0.12106666962305705
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,float16,0,0.038015998899936676
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,float16,0,0.06267199913660686
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,float16,0,0.1218293309211731
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,float16,0,0.037791999677817024
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,float16,0,0.027114666998386383
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,float16,0,0.027119999130566914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,float16,0,0.024090667565663654
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,float16,0,0.023978665471076965
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,float16,0,0.2208319902420044
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,float16,0,0.21968533595403036
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,float16,0,0.02218666672706604
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,float16,0,0.020848001043001812
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,float16,0,0.022245332598686218
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,float16,0,0.02085866779088974
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,float16,0,0.021642667551835377
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,float16,0,0.0321066677570343
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,float16,0,0.032111999889214836
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,float16,0,0.0529120018084844
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,float16,0,0.052655999859174095
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,float16,0,0.021615999440352123
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,float16,0,0.018426666657129925
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,float16,0,0.01846933364868164
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,float16,0,0.016778666526079178
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,float16,0,0.10648533701896667
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,float16,0,0.016879999389251072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,float16,0,0.015386667102575302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,float16,0,0.015135999768972397
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,float16,0,0.015802666544914246
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,float16,0,0.10685867071151733
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,float16,0,0.015210667004187902
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,float16,0,0.015546667079130808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,float16,0,0.018437333405017853
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,float16,0,0.01865600049495697
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,float16,0,0.015471999843915304
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,float16,0,0.02712533374627431
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,float16,0,0.027093333502610523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,float16,0,0.013829333086808523
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,float16,0,0.01381333296497663
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,float16,0,0.012245333443085352
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,float16,0,0.011909333368142446
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,float16,0,0.011887999872366587
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,float16,0,0.01180800050497055
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,float16,0,0.011957333733638128
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,float16,0,0.04872000217437744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,float16,0,0.048341333866119385
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,float16,0,0.013594667116800943
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,float16,0,0.013674666484196981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,float16,0,0.015247999380032221
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,float16,0,0.018053332964579265
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,float16,0,0.01515199989080429
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,float16,0,0.01850133389234543
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,float16,0,0.01219733307758967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,float16,0,0.012122667084137598
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,float16,0,0.02678400029738744
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,float16,0,0.011770666887362799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,float16,0,0.026848000784715016
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,float16,0,0.011834666132926941
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,float16,0,0.011744000017642975
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,float16,0,0.011855999628702799
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,float16,0,0.011402666568756104
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,float16,0,0.011541333049535751
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,float16,0,0.01339200014869372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,float16,0,0.011999999483426413
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,float16,0,0.013253333667914072
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,float16,0,0.01526933287580808
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,float16,0,0.015263999501864115
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,float16,0,0.01219733307758967
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,float16,0,0.011727999895811081
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,float16,0,0.011877333124478659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,float16,0,0.011525332927703857
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,float16,0,0.020453333854675293
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,float16,0,0.011519999553759893
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,float16,0,0.011221333096424738
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,float16,0,0.02088533341884613
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,float16,0,0.011306667079528173
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,float16,0,0.011893333246310553
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,float16,0,0.011989332735538483
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,float16,0,0.012319999436537424
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,float16,0,0.013674666484196981
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,float16,0,0.013493333011865616
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,float16,0,0.011477333803971609
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,float16,0,0.011877333124478659
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,float16,0,0.011493333925803503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,float16,0,0.011551999797423681
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,float16,0,0.017743999759356182
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,float16,0,0.011301333705584208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,float16,0,0.01138666644692421
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,float16,0,0.011328000575304031
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,float16,0,0.017610666652520496
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,float16,0,0.01101333275437355
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,float16,0,0.01097600037852923
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,0,0.011301333705584208
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,0,0.01109333336353302
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,0,0.01119999960064888
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,0,0.011472000430027643
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,0,0.011231999844312668
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,0,0.010794666906197866
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,0,0.010698666175206503
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,0,0.011109333485364914
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,0,0.010954666882753372
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,0,0.010687999427318573
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,0,0.013978666315476099
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,0,0.013909333695967993
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,0,0.010751999914646149
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,0,0.01073066641887029
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,0,0.010677333921194077
VLLM,0.14.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,0,0.010879999647537867
