framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,4,128,1,float16,float16,0,96.93115234375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,2,128,1,float16,float16,0,96.1239013671875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,64,128,1,float16,float16,0,97.05284627278645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,1,128,1,float16,float16,0,95.54551188151042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,64,128,1,float16,float16,0,49.03355916341146
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,64,8,128,1,float16,float16,0,193.55989583333334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,1,128,1,float16,float16,0,47.311187744140625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,64,2,128,1,float16,float16,0,190.68314615885416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,2,128,1,float16,float16,0,47.7675526936849
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,64,1,128,1,float16,float16,0,192.79207356770834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,64,8,128,1,float16,float16,0,97.06325276692708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,64,128,1,float16,float16,0,24.432640075683594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,1,128,1,float16,float16,0,23.89452870686849
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,2,128,1,float16,float16,0,24.02594248453776
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,4,128,1,float16,float16,0,48.55756632486979
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,4,128,1,float16,float16,0,24.613375345865887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,64,8,128,1,float16,float16,0,48.552276611328125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,64,8,128,1,float16,float16,0,24.196436564127605
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,64,4,128,1,float16,float16,0,192.22904459635416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,64,128,1,float16,float16,0,56.539306640625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,1,128,1,float16,float16,0,54.548482259114586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,2,128,1,float16,float16,0,54.75328063964844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,1,128,1,float16,float16,0,27.26604715983073
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,64,1,128,1,float16,float16,0,107.0757548014323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,64,128,1,float16,float16,0,28.33459218343099
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,4,128,1,float16,float16,0,54.538411458333336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,64,2,128,1,float16,float16,0,108.4376729329427
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,64,4,128,1,float16,float16,0,109.7171630859375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,64,8,128,1,float16,float16,0,110.0782979329427
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,64,128,1,float16,float16,0,14.069077809651693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,1,128,1,float16,float16,0,13.563563028971354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,2,128,1,float16,float16,0,13.670570373535156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,64,8,128,1,float16,float16,0,55.12669881184896
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,4,128,1,float16,float16,0,13.754197438557943
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,4,128,1,float16,float16,0,27.087018330891926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,2,128,1,float16,float16,0,27.14214324951172
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,64,8,128,1,float16,float16,0,27.6312739054362
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,64,8,128,1,float16,float16,0,13.592917124430338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,1,128,1,float16,float16,0,38.13717397054037
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,64,128,1,float16,float16,0,39.02583567301432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,2,128,1,float16,float16,0,38.27848561604818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,4,128,1,float16,float16,0,38.19571177164713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,64,128,1,float16,float16,0,19.53638458251953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,1,128,1,float16,float16,0,19.137364705403645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,2,128,1,float16,float16,0,18.93222427368164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,64,1,128,1,float16,float16,0,75.47955322265625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,64,2,128,1,float16,float16,0,76.28697713216145
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,64,4,128,1,float16,float16,0,77.46201578776042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,64,128,1,float16,float16,0,9.946965535481771
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,64,8,128,1,float16,float16,0,38.61213938395182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,64,8,128,1,float16,float16,0,77.06709289550781
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,1,128,1,float16,float16,0,9.372842788696289
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,4,128,1,float16,float16,0,18.979668935139973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,2,128,1,float16,float16,0,9.46602694193522
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,64,8,128,1,float16,float16,0,19.307348887125652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,8,128,1,float16,float16,0,9.513130823771158
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,64,4,128,1,float16,float16,0,9.565184275309244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,1,128,1,float16,float16,0,49.39093526204427
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,64,128,1,float16,float16,0,51.48859659830729
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,2,128,1,float16,float16,0,48.950785319010414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,4,128,1,float16,float16,0,49.90446980794271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,1,128,1,float16,float16,0,24.69512430826823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,64,128,1,float16,float16,0,25.8515625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,2,128,1,float16,float16,0,25.291605631510418
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,64,1,128,1,float16,float16,0,98.9317118326823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,64,2,128,1,float16,float16,0,98.74500528971355
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,64,128,1,float16,float16,0,13.051563262939453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,64,4,128,1,float16,float16,0,99.0967508951823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,64,8,128,1,float16,float16,0,98.605224609375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,64,8,128,1,float16,float16,0,50.10705057779948
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,4,128,1,float16,float16,0,24.745473225911457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,64,8,128,1,float16,float16,0,24.769877115885418
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,64,128,1,float16,float16,0,6.341461181640625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,1,128,1,float16,float16,0,6.250325520833333
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,1,128,1,float16,float16,0,12.256426493326822
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,2,128,1,float16,float16,0,12.381866455078125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,4,128,1,float16,float16,0,12.533589680989584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,64,8,128,1,float16,float16,0,12.251818339029947
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,2,128,1,float16,float16,0,6.076245625813802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,4,128,1,float16,float16,0,6.048255920410156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,64,8,128,1,float16,float16,0,6.050474802652995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,1,128,1,float16,float16,0,27.82634735107422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,64,128,1,float16,float16,0,29.578580220540363
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,2,128,1,float16,float16,0,28.2250239054362
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,4,128,1,float16,float16,0,28.325716654459637
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,2,128,1,float16,float16,0,13.814442952473959
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,1,128,1,float16,float16,0,13.920597076416016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,64,128,1,float16,float16,0,14.737237294514975
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,64,1,128,1,float16,float16,0,56.15906270345052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,64,2,128,1,float16,float16,0,56.49698384602865
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,64,128,1,float16,float16,0,7.32689094543457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,64,4,128,1,float16,float16,0,57.19347127278646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,64,8,128,1,float16,float16,0,57.61518859863281
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,64,8,128,1,float16,float16,0,28.57557423909505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,4,128,1,float16,float16,0,14.087509155273438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,64,128,1,float16,float16,0,3.7007360458374023
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,1,128,1,float16,float16,0,6.895957310994466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,64,8,128,1,float16,float16,0,14.20236841837565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,1,128,1,float16,float16,0,3.493376096089681
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,2,128,1,float16,float16,0,6.930432001749675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,4,128,1,float16,float16,0,6.924458821614583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,64,8,128,1,float16,float16,0,7.009621302286784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,2,128,1,float16,float16,0,3.486207962036133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,4,128,1,float16,float16,0,3.463850657145182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,64,8,128,1,float16,float16,0,3.5172694524129233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,2,128,1,float16,float16,0,26.137771606445312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,64,128,1,float16,float16,0,27.756032307942707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,1,128,1,float16,float16,0,25.99560546875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,1,128,1,float16,float16,0,12.972544352213541
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,64,128,1,float16,float16,0,13.894485473632812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,64,2,128,1,float16,float16,0,51.61284383138021
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,64,1,128,1,float16,float16,0,51.35718282063802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,64,4,128,1,float16,float16,0,52.167338053385414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,4,128,1,float16,float16,0,26.271743774414062
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,64,8,128,1,float16,float16,0,52.829182942708336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,64,8,128,1,float16,float16,0,26.414591471354168
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,2,128,1,float16,float16,0,12.914005279541016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,4,128,1,float16,float16,0,13.025450388590494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,1,128,1,float16,float16,0,6.130346934000651
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,2,128,1,float16,float16,0,6.199808120727539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,64,128,1,float16,float16,0,6.838954925537109
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,4,128,1,float16,float16,0,6.397269566853841
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,64,128,1,float16,float16,0,3.4510507583618164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,1,128,1,float16,float16,0,3.122346560160319
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,2,128,1,float16,float16,0,3.140437444051107
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,64,8,128,1,float16,float16,0,6.445397059122722
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,64,8,128,1,float16,float16,0,13.065728505452475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,4,128,1,float16,float16,0,3.1870292027791343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,64,128,1,float16,float16,0,1.7525760332743328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,64,8,128,1,float16,float16,0,3.1955626805623374
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,1,128,1,float16,float16,0,1.5989759763081868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,2,128,1,float16,float16,0,1.5897599856058757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,4,128,1,float16,float16,0,1.6141653060913086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,64,8,128,1,float16,float16,0,1.6523946126302083
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,2,128,1,float16,float16,0,14.866261800130209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,1,128,1,float16,float16,0,14.755669911702475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,4,128,1,float16,float16,0,15.373483022054037
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,64,128,1,float16,float16,0,16.80520502726237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,1,128,1,float16,float16,0,7.269205093383789
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,64,128,1,float16,float16,0,8.109909057617188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,64,1,128,1,float16,float16,0,29.71630859375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,64,2,128,1,float16,float16,0,30.019070943196613
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,2,128,1,float16,float16,0,7.320575714111328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,64,4,128,1,float16,float16,0,30.749183654785156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,64,8,128,1,float16,float16,0,30.64575958251953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,64,8,128,1,float16,float16,0,15.154688517252604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,4,128,1,float16,float16,0,7.562410354614258
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,1,128,1,float16,float16,0,3.5869013468424478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,64,128,1,float16,float16,0,3.9990612665812173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,2,128,1,float16,float16,0,3.621546745300293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,4,128,1,float16,float16,0,3.7560319900512695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,64,8,128,1,float16,float16,0,7.422122955322266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,64,128,1,float16,float16,0,2.0375893910725913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,64,8,128,1,float16,float16,0,3.7149012883504233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,1,128,1,float16,float16,0,1.798485279083252
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,2,128,1,float16,float16,0,1.8483200073242188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,64,128,1,float16,float16,0,1.0550613403320312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,2,128,1,float16,float16,0,0.9301333427429199
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,4,128,1,float16,float16,0,0.9700693289438883
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,1,128,1,float16,float16,0,0.9412266413370768
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,64,8,128,1,float16,float16,0,0.9572693506876627
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,4,128,1,float16,float16,0,1.8551467259724934
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,64,8,128,1,float16,float16,0,1.8648746808369954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,1,128,1,float16,float16,0,13.93288548787435
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,4,128,1,float16,float16,0,14.68893814086914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,2,128,1,float16,float16,0,13.96872584025065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,64,128,1,float16,float16,0,16.249685923258465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,1,128,1,float16,float16,0,6.770175933837891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,64,128,1,float16,float16,0,8.119978586832682
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,2,128,1,float16,float16,0,6.788096110026042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,64,8,128,1,float16,float16,0,29.358421325683594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,64,1,128,1,float16,float16,0,28.12501271565755
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,64,2,128,1,float16,float16,0,28.30096944173177
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,64,8,128,1,float16,float16,0,14.577835083007812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,1,128,1,float16,float16,0,3.355135917663574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,64,128,1,float16,float16,0,3.9423999786376953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,64,4,128,1,float16,float16,0,28.91161600748698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,4,128,1,float16,float16,0,6.965077082316081
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,64,8,128,1,float16,float16,0,7.110656102498372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,2,128,1,float16,float16,0,3.3389228185017905
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,8,128,1,float16,float16,0,3.4877440134684243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,64,4,128,1,float16,float16,0,3.463168144226074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,1,128,1,float16,float16,0,1.6549545923868816
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,64,128,1,float16,float16,0,1.981610616048177
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,4,128,1,float16,float16,0,1.7648639678955078
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,8,128,1,float16,float16,0,1.7890987396240234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,2,128,1,float16,float16,0,0.8763733704884847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,1,128,1,float16,float16,0,0.8606719970703125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,4,128,1,float16,float16,0,0.8905386924743652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,64,2,128,1,float16,float16,0,1.6841386159261067
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,64,128,1,float16,float16,0,1.0226346651713054
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,64,128,1,float16,float16,0,0.5307733217875162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,64,8,128,1,float16,float16,0,0.8934400081634521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,1,128,1,float16,float16,0,0.45875199635823566
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,2,128,1,float16,float16,0,0.45482667287190753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,8,128,1,float16,float16,0,0.47274665037790936
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,64,4,128,1,float16,float16,0,0.46557867527008057
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,1,128,1,float16,float16,0,8.184832255045572
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,4,128,1,float16,float16,0,8.527701059977213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,2,128,1,float16,float16,0,8.139434814453125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,64,128,1,float16,float16,0,4.84983476003011
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,64,128,1,float16,float16,0,9.993386586507162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,64,4,128,1,float16,float16,0,17.333248138427734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,64,8,128,1,float16,float16,0,17.520640055338543
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,64,8,128,1,float16,float16,0,8.69324811299642
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,1,128,1,float16,float16,0,3.879253387451172
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,64,1,128,1,float16,float16,0,16.46899159749349
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,64,128,1,float16,float16,0,2.450431982676188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,64,2,128,1,float16,float16,0,17.014272054036457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,4,128,1,float16,float16,0,4.14685853322347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,8,128,1,float16,float16,0,4.1750186284383135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,1,128,1,float16,float16,0,1.9467946688334148
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,64,2,128,1,float16,float16,0,3.920725186665853
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,4,128,1,float16,float16,0,2.0497066179911294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,64,128,1,float16,float16,0,1.2127573490142822
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,2,128,1,float16,float16,0,1.0071039994557698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,1,128,1,float16,float16,0,1.0042026837666829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,8,128,1,float16,float16,0,1.049770673116048
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,64,128,1,float16,float16,0,0.6272000074386597
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,1,128,1,float16,float16,0,0.5154133240381876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,2,128,1,float16,float16,0,1.972394625345866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,2,128,1,float16,float16,0,0.5213866631189982
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,64,4,128,1,float16,float16,0,1.0564266840616863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,4,128,1,float16,float16,0,0.5398186842600504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,2,128,1,float16,float16,0,0.2815999984741211
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,64,128,1,float16,float16,0,0.3324586749076843
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,64,8,128,1,float16,float16,0,0.5452800194422404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,4,128,1,float16,float16,0,0.28945066531499225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,8,128,1,float16,float16,0,0.2913279930750529
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,64,1,128,1,float16,float16,0,0.2759679953257243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,64,8,128,1,float16,float16,0,2.063360055287679
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,64,128,1,float16,float16,0,10.388992309570312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,4,128,1,float16,float16,0,8.172202428181967
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,1,128,1,float16,float16,0,7.89742914835612
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,64,1,128,1,float16,float16,0,16.18141810099284
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,64,128,1,float16,float16,0,5.033983866373698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,1,128,1,float16,float16,0,3.7986987431844077
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,8,128,1,float16,float16,0,8.504661560058594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,2,128,1,float16,float16,0,3.918506622314453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,64,2,128,1,float16,float16,0,8.033962885538736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,64,2,128,1,float16,float16,0,16.516778310139973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,1,128,1,float16,float16,0,1.9056639671325684
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,64,128,1,float16,float16,0,2.5180160204569497
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,2,128,1,float16,float16,0,2.0626773834228516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,4,128,1,float16,float16,0,4.061866760253906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,64,8,128,1,float16,float16,0,4.076032002766927
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,4,128,1,float16,float16,0,2.092202663421631
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,64,8,128,1,float16,float16,0,2.045269330342611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,64,8,128,1,float16,float16,0,17.48479970296224
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,1,128,1,float16,float16,0,0.9552213350931803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,64,128,1,float16,float16,0,1.2612266540527344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,2,128,1,float16,float16,0,0.9977173010508219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,4,128,1,float16,float16,0,1.0316800276438396
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,64,8,128,1,float16,float16,0,1.0408960183461506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,64,128,1,float16,float16,0,0.6425600051879883
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,2,128,1,float16,float16,0,0.5073920090993246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,4,128,1,float16,float16,0,0.529749313990275
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,1,128,1,float16,float16,0,0.2616320053736369
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,4,128,1,float16,float16,0,0.28091732660929364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,2,128,1,float16,float16,0,0.26385066906611127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,8,128,1,float16,float16,0,0.2792106668154399
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,8,128,1,float16,float16,0,0.532309333483378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,64,64,128,1,float16,float16,0,0.3357013463973999
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,64,128,1,float16,float16,0,0.18295466899871826
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,1,128,1,float16,float16,0,0.14677332838376364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,2,128,1,float16,float16,0,0.1474560002485911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,4,128,1,float16,float16,0,0.15377066532770792
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,64,8,128,1,float16,float16,0,0.15496533115704855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,64,1,128,1,float16,float16,0,0.49032533168792725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,64,4,128,1,float16,float16,0,17.16326395670573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,1,128,1,float16,float16,0,4.889087994893392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,64,128,1,float16,float16,0,7.391232172648112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,4,128,1,float16,float16,0,5.4103037516276045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,64,128,1,float16,float16,0,3.6684799194335938
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,2,128,1,float16,float16,0,5.105493227640788
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,64,1,128,1,float16,float16,0,10.179072062174479
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,1,128,1,float16,float16,0,2.3961599667867026
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,4,128,1,float16,float16,0,2.7450027465820312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,64,128,1,float16,float16,0,1.808725357055664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,64,8,128,1,float16,float16,0,5.5903574625651045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,64,2,128,1,float16,float16,0,10.50709342956543
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,2,128,1,float16,float16,0,2.4934399922688804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,64,8,128,1,float16,float16,0,2.914133389790853
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,64,4,128,1,float16,float16,0,11.152725219726562
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,64,8,128,1,float16,float16,0,11.431935628255209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,1,128,1,float16,float16,0,1.1801599661509197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,2,128,1,float16,float16,0,1.237162669499715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,64,128,1,float16,float16,0,0.9161386489868164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,64,128,1,float16,float16,0,0.46882132689158124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,8,128,1,float16,float16,0,1.3516799608866374
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,8,128,1,float16,float16,0,0.6869333585103353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,1,128,1,float16,float16,0,0.3128319978713989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,2,128,1,float16,float16,0,0.32494932413101196
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,64,128,1,float16,float16,0,0.2455893357594808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,1,128,1,float16,float16,0,0.16759467124938965
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,1,128,1,float16,float16,0,0.6014293432235718
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,8,128,1,float16,float16,0,0.3541333278020223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,2,128,1,float16,float16,0,0.17459199825922647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,4,128,1,float16,float16,0,0.18636800845464072
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,64,4,128,1,float16,float16,0,0.34867199261983234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,64,8,128,1,float16,float16,0,0.18653867642084757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,2,128,1,float16,float16,0,0.09762133161226909
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,1,128,1,float16,float16,0,0.09523199995358785
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,64,128,1,float16,float16,0,0.1327786644299825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,4,128,1,float16,float16,0,0.10222933689753215
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,64,8,128,1,float16,float16,0,0.10461866855621338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,64,128,1,float16,float16,0,0.07833600044250488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,1,128,1,float16,float16,0,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,2,128,1,float16,float16,0,0.05580799778302511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,4,128,1,float16,float16,0,0.05870933334032694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,64,8,128,1,float16,float16,0,0.05853866537412008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,64,4,128,1,float16,float16,0,1.3404159545898438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,2,128,1,float16,float16,0,0.6285653511683146
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,64,4,128,1,float16,float16,0,0.6807893117268881
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,64,128,1,float16,float16,0,2.9207894007364907
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,64,2,128,1,float16,float16,0,3.625472068786621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,1,128,1,float16,float16,0,1.6889173189798992
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,64,1,128,1,float16,float16,0,3.4491732915242515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,2,128,1,float16,float16,0,1.7949013710021973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,64,4,128,1,float16,float16,0,4.044970512390137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,4,128,1,float16,float16,0,0.9833813508351644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,1,128,1,float16,float16,0,0.8212479750315348
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,2,128,1,float16,float16,0,0.8944640159606934
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,64,128,1,float16,float16,0,1.4648319880167644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,8,128,1,float16,float16,0,2.0348587036132812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,64,4,128,1,float16,float16,0,1.997312068939209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,64,8,128,1,float16,float16,0,1.0118827025095622
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,1,128,1,float16,float16,0,0.4179626703262329
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,64,128,1,float16,float16,0,0.7424000104268392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,2,128,1,float16,float16,0,0.4490240017573039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,4,128,1,float16,float16,0,0.5012480020523071
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,2,128,1,float16,float16,0,0.23040000597635904
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,64,128,1,float16,float16,0,0.385535995165507
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,4,128,1,float16,float16,0,0.2568533420562744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,64,8,128,1,float16,float16,0,4.062890688578288
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,1,128,1,float16,float16,0,0.11724799871444702
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,8,128,1,float16,float16,0,0.26385066906611127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,64,1,128,1,float16,float16,0,0.2208426594734192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,64,128,1,float16,float16,0,0.2034346659978231
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,64,8,128,1,float16,float16,0,0.5075626770655314
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,2,128,1,float16,float16,0,0.12339199582735698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,2,128,1,float16,float16,0,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,4,128,1,float16,float16,0,0.13636266191800436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,64,128,1,float16,float16,0,0.1063253382841746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,4,128,1,float16,float16,0,0.07560533285140991
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,1,128,1,float16,float16,0,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,64,128,1,float16,float16,0,0.06297599772612254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,1,128,1,float16,float16,0,0.039594667653242745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,2,128,1,float16,float16,0,0.039936001102129616
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,64,8,128,1,float16,float16,0,0.0766293356815974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,8,128,1,float16,float16,0,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,64,4,128,1,float16,float16,0,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,1,128,1,float16,float16,0,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,2,128,1,float16,float16,0,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,64,128,1,float16,float16,0,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,4,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,64,8,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,64,8,128,1,float16,float16,0,0.14011733730634054
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,2,128,1,float16,float16,0,0.6900053024291992
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,1,128,1,float16,float16,0,0.637440005938212
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,64,128,1,float16,float16,0,1.405610720316569
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,64,1,128,1,float16,float16,0,1.5037439664204915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,64,128,1,float16,float16,0,0.7080960273742676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,1,128,1,float16,float16,0,0.31965865691502887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,2,128,1,float16,float16,0,0.34406399726867676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,4,128,1,float16,float16,0,0.8026453653971354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,64,8,128,1,float16,float16,0,0.8149333000183105
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,64,2,128,1,float16,float16,0,1.6018773714701335
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,64,8,128,1,float16,float16,0,1.8136746088663738
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,64,4,128,1,float16,float16,0,1.835349400838216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,64,128,1,float16,float16,0,0.3582293192545573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,4,128,1,float16,float16,0,0.40379734834035236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,1,128,1,float16,float16,0,0.16827734311421713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,1,128,1,float16,float16,0,0.09250133236249287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,4,128,1,float16,float16,0,0.20616533358891806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,64,8,128,1,float16,float16,0,0.4145493507385254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,2,128,1,float16,float16,0,0.17937066157658896
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,4,128,1,float16,float16,0,0.11178666353225708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,64,128,1,float16,float16,0,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,8,128,1,float16,float16,0,0.11417599519093831
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,64,2,128,1,float16,float16,0,0.09727999567985535
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,2,128,1,float16,float16,0,0.05649066468079885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,64,128,1,float16,float16,0,0.10427733262379964
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,64,8,128,1,float16,float16,0,0.20889600118001303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,1,128,1,float16,float16,0,0.055125330885251365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,2,128,1,float16,float16,0,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,4,128,1,float16,float16,0,0.06348800162474315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,1,128,1,float16,float16,0,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,1,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,64,128,1,float16,float16,0,0.058880001306533813
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,4,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,64,8,128,1,float16,float16,0,0.0653653343518575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,2,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,4,128,1,float16,float16,0,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,64,128,1,float16,float16,0,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,64,8,128,1,float16,float16,0,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,64,8,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,1,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,4,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,64,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,8,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,64,2,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,2,128,1,float16,float16,0,0.3394560019175212
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,1,128,1,float16,float16,0,0.31436800956726074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,64,1,128,1,float16,float16,0,0.6167893409729004
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,64,4,128,1,float16,float16,0,0.7796053091684977
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,4,128,1,float16,float16,0,0.3949226538340251
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,8,128,1,float16,float16,0,0.40618666013081867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,64,128,1,float16,float16,0,0.35089067618052167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,64,2,128,1,float16,float16,0,0.6758399804433187
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,2,128,1,float16,float16,0,0.1795413295427958
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,8,128,1,float16,float16,0,0.20923733711242676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,64,64,128,1,float16,float16,0,0.6896639664967855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,1,128,1,float16,float16,0,0.16657066345214844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,2,128,1,float16,float16,0,0.09762133161226909
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,64,128,1,float16,float16,0,0.1853440006573995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,64,4,128,1,float16,float16,0,0.20548266172409058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,4,128,1,float16,float16,0,0.1104213297367096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,64,8,128,1,float16,float16,0,0.7966720263163248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,1,128,1,float16,float16,0,0.053077335158983864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,64,128,1,float16,float16,0,0.09915733337402344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,8,128,1,float16,float16,0,0.06400000055631001
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,1,128,1,float16,float16,0,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,4,128,1,float16,float16,0,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,4,128,1,float16,float16,0,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,8,128,1,float16,float16,0,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,2,128,1,float16,float16,0,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,64,2,128,1,float16,float16,0,0.05614933371543884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,64,128,1,float16,float16,0,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,1,128,1,float16,float16,0,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,1,128,1,float16,float16,0,0.09301333626111348
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,2,128,1,float16,float16,0,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,4,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,64,8,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,64,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,4,128,1,float16,float16,0,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,8,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,64,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,2,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,64,1,128,1,float16,float16,0,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,64,8,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,64,64,128,1,float16,float16,0,0.05717333157857259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,64,8,128,1,float16,float16,0,0.1129813293615977
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,1,128,1,float16,float16,0,0.29661866029103595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,2,128,1,float16,float16,0,0.3092479904492696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,8,128,1,float16,float16,0,0.33433600266774494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,4,128,1,float16,float16,0,0.33348266283671063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,64,1,128,1,float16,float16,0,0.5824853181838989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,64,2,128,1,float16,float16,0,0.6038186550140381
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,2,128,1,float16,float16,0,0.16025599837303162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,1,128,1,float16,float16,0,0.1539413332939148
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,64,64,128,1,float16,float16,0,0.44868266582489014
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,64,128,1,float16,float16,0,0.22937599817911783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,64,4,128,1,float16,float16,0,0.6517759958902994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,1,128,1,float16,float16,0,0.08328533172607422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,64,8,128,1,float16,float16,0,0.6568959951400757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,64,128,1,float16,float16,0,0.12168533603350322
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,64,128,1,float16,float16,0,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,4,128,1,float16,float16,0,0.1718613306681315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,2,128,1,float16,float16,0,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,4,128,1,float16,float16,0,0.04625066618124644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,64,8,128,1,float16,float16,0,0.1728853384653727
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,4,128,1,float16,float16,0,0.09198932846387227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,64,128,1,float16,float16,0,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,1,128,1,float16,float16,0,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,8,128,1,float16,float16,0,0.04642133414745331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,4,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,2,128,1,float16,float16,0,0.02611200014750163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,64,128,1,float16,float16,0,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,1,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,64,8,128,1,float16,float16,0,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,8,128,1,float16,float16,0,0.09318400422732036
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,4,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,64,8,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,64,1,128,1,float16,float16,0,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,1,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,8,128,1,float16,float16,0,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,64,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,64,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,64,2,128,1,float16,float16,0,0.08669867118199666
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,2,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,64,4,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,64,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,1,128,1,float16,float16,0,0.28381866216659546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,4,128,1,float16,float16,0,0.3020799954732259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,2,128,1,float16,float16,0,0.28996266921361286
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,1,128,1,float16,float16,0,0.14591999848683676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,64,128,1,float16,float16,0,0.18210132916768393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,64,2,128,1,float16,float16,0,0.5765120188395182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,64,128,1,float16,float16,0,0.3573760191599528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,64,8,128,1,float16,float16,0,0.3041279911994934
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,64,1,128,1,float16,float16,0,0.5645653406778971
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,64,4,128,1,float16,float16,0,0.6019413471221924
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,2,128,1,float16,float16,0,0.14882133404413858
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,64,8,128,1,float16,float16,0,0.6036479870478312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,4,128,1,float16,float16,0,0.155648003021876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,1,128,1,float16,float16,0,0.07321600119272868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,4,128,1,float16,float16,0,0.0766293356815974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,64,8,128,1,float16,float16,0,0.1565013329188029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,1,128,1,float16,float16,0,0.04095999896526337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,2,128,1,float16,float16,0,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,64,128,1,float16,float16,0,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,64,128,1,float16,float16,0,0.09864532947540283
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,2,128,1,float16,float16,0,0.0747519979874293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,8,128,1,float16,float16,0,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,64,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,64,8,128,1,float16,float16,0,0.07850666840871175
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,4,128,1,float16,float16,0,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,64,4,128,1,float16,float16,0,0.042837331692377724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,8,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,64,128,1,float16,float16,0,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,1,128,1,float16,float16,0,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,1,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,64,2,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,2,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,4,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,64,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,1,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,64,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,2,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,4,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,64,8,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,64,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,64,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,64,128,1,float16,float16,0,0.2512213389078776
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,0,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,0,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,0,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,0,0.09779199957847595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,0,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,0,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,64,128,1,float16,float16,0,0.13038933277130127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,0,0.05649066468079885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,0,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,0,0.03583999971548716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,64,128,1,float16,float16,0,0.06997333467006683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,0,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,64,128,1,float16,float16,0,0.04095999896526337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,0,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,0,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,64,128,1,float16,float16,0,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,0,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,64,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,64,128,1,float16,float16,0,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,64,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,2,128,1,float16,float16,0,71.30419413248698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,1,128,1,float16,float16,0,72.33160400390625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,4,128,1,float16,float16,0,73.09448750813802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,48,128,1,float16,float16,0,73.6699727376302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,2,128,1,float16,float16,0,36.053845723470054
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,1,128,1,float16,float16,0,36.349952697753906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,48,128,1,float16,float16,0,37.54837290445963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,48,128,1,float16,float16,0,18.763263702392578
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,48,1,128,1,float16,float16,0,144.13618977864584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,48,2,128,1,float16,float16,0,145.40082804361978
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,48,4,128,1,float16,float16,0,145.23391723632812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,48,8,128,1,float16,float16,0,146.62861124674478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,48,8,128,1,float16,float16,0,72.27084859212239
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,4,128,1,float16,float16,0,36.25198872884115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,48,8,128,1,float16,float16,0,36.637524922688804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,1,128,1,float16,float16,0,18.2826665242513
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,2,128,1,float16,float16,0,18.18726348876953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,4,128,1,float16,float16,0,18.279253641764324
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,48,8,128,1,float16,float16,0,18.33403778076172
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,48,128,1,float16,float16,0,42.826924641927086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,1,128,1,float16,float16,0,40.739156087239586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,4,128,1,float16,float16,0,40.98235829671224
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,2,128,1,float16,float16,0,41.22112019856771
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,1,128,1,float16,float16,0,20.608853658040363
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,48,128,1,float16,float16,0,21.302783966064453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,48,2,128,1,float16,float16,0,81.88142903645833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,48,1,128,1,float16,float16,0,82.71701558430989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,48,4,128,1,float16,float16,0,82.69346110026042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,48,8,128,1,float16,float16,0,83.19488016764323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,48,128,1,float16,float16,0,10.57041041056315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,48,8,128,1,float16,float16,0,41.99714152018229
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,1,128,1,float16,float16,0,10.391722361246744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,2,128,1,float16,float16,0,20.328960418701172
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,4,128,1,float16,float16,0,20.909056345621746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,48,8,128,1,float16,float16,0,20.601343790690105
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,2,128,1,float16,float16,0,10.303317387898764
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,4,128,1,float16,float16,0,10.384554545084635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,48,8,128,1,float16,float16,0,10.25433603922526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,1,128,1,float16,float16,0,28.681045532226562
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,48,128,1,float16,float16,0,29.662208557128906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,2,128,1,float16,float16,0,28.9435297648112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,4,128,1,float16,float16,0,28.656982421875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,1,128,1,float16,float16,0,14.309717814127604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,48,128,1,float16,float16,0,14.895615895589193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,2,128,1,float16,float16,0,14.141780853271484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,48,2,128,1,float16,float16,0,57.92921447753906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,48,1,128,1,float16,float16,0,58.23180643717448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,48,128,1,float16,float16,0,7.3821868896484375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,48,8,128,1,float16,float16,0,58.22583516438802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,48,8,128,1,float16,float16,0,28.889429728190105
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,48,4,128,1,float16,float16,0,57.43718465169271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,1,128,1,float16,float16,0,7.298559824625651
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,2,128,1,float16,float16,0,7.2618668874104815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,4,128,1,float16,float16,0,14.496938069661459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,48,8,128,1,float16,float16,0,14.543018341064453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,4,128,1,float16,float16,0,7.103658676147461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,48,8,128,1,float16,float16,0,7.288319905598958
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,1,128,1,float16,float16,0,36.98773447672526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,48,128,1,float16,float16,0,38.97924296061198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,2,128,1,float16,float16,0,36.721153259277344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,4,128,1,float16,float16,0,37.71989440917969
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,1,128,1,float16,float16,0,18.269524892171223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,48,128,1,float16,float16,0,19.343359629313152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,2,128,1,float16,float16,0,18.63714090983073
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,48,1,128,1,float16,float16,0,74.23931884765625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,48,2,128,1,float16,float16,0,73.52524820963542
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,48,128,1,float16,float16,0,9.76913070678711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,48,4,128,1,float16,float16,0,76.00230407714844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,48,8,128,1,float16,float16,0,76.27827453613281
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,48,8,128,1,float16,float16,0,37.658111572265625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,4,128,1,float16,float16,0,18.777088165283203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,48,8,128,1,float16,float16,0,18.87761052449544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,48,128,1,float16,float16,0,4.835839907328288
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,1,128,1,float16,float16,0,9.176575978597006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,1,128,1,float16,float16,0,4.633088111877441
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,2,128,1,float16,float16,0,9.12452252705892
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,4,128,1,float16,float16,0,9.398613611857096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,2,128,1,float16,float16,0,4.55236275990804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,48,8,128,1,float16,float16,0,9.459199905395508
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,4,128,1,float16,float16,0,4.6259199778238935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,48,8,128,1,float16,float16,0,4.73088010152181
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,1,128,1,float16,float16,0,21.15771738688151
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,2,128,1,float16,float16,0,21.40330759684245
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,48,128,1,float16,float16,0,22.252031962076824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,4,128,1,float16,float16,0,21.880320231119793
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,1,128,1,float16,float16,0,10.39957364400228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,48,128,1,float16,float16,0,11.214847564697266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,2,128,1,float16,float16,0,10.71786626180013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,48,1,128,1,float16,float16,0,42.25006866455078
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,48,2,128,1,float16,float16,0,42.67741902669271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,48,4,128,1,float16,float16,0,42.72486368815104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,48,128,1,float16,float16,0,5.63968022664388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,48,8,128,1,float16,float16,0,43.25085957845052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,48,8,128,1,float16,float16,0,21.722793579101562
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,4,128,1,float16,float16,0,10.623658498128256
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,1,128,1,float16,float16,0,5.219327926635742
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,48,128,1,float16,float16,0,2.8023465474446616
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,2,128,1,float16,float16,0,5.2111358642578125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,4,128,1,float16,float16,0,5.239808082580566
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,48,8,128,1,float16,float16,0,10.835285186767578
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,48,8,128,1,float16,float16,0,5.418495814005534
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,1,128,1,float16,float16,0,2.6603520711263022
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,2,128,1,float16,float16,0,2.612394650777181
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,4,128,1,float16,float16,0,2.6741758982340493
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,48,8,128,1,float16,float16,0,2.645674705505371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,1,128,1,float16,float16,0,19.24181365966797
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,2,128,1,float16,float16,0,19.569664001464844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,4,128,1,float16,float16,0,20.108800252278645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,48,128,1,float16,float16,0,21.230592091878254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,1,128,1,float16,float16,0,9.64676284790039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,48,128,1,float16,float16,0,10.540032068888346
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,2,128,1,float16,float16,0,9.737557093302408
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,48,1,128,1,float16,float16,0,38.656341552734375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,48,2,128,1,float16,float16,0,38.59080505371094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,48,4,128,1,float16,float16,0,39.97661844889323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,48,128,1,float16,float16,0,5.227690696716309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,48,8,128,1,float16,float16,0,40.50005340576172
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,48,8,128,1,float16,float16,0,19.87225596110026
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,4,128,1,float16,float16,0,9.995775858561197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,1,128,1,float16,float16,0,4.654421488444011
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,48,8,128,1,float16,float16,0,9.89747174580892
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,2,128,1,float16,float16,0,4.680533409118652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,1,128,1,float16,float16,0,2.3642452557881675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,48,128,1,float16,float16,0,2.6625706354777017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,4,128,1,float16,float16,0,4.841301282246907
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,2,128,1,float16,float16,0,2.4396799405415854
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,48,8,128,1,float16,float16,0,4.805802663167317
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,1,128,1,float16,float16,0,1.2398933569590251
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,48,128,1,float16,float16,0,1.3315412998199463
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,8,128,1,float16,float16,0,2.480639934539795
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,48,4,128,1,float16,float16,0,2.456746737162272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,2,128,1,float16,float16,0,1.2518400351206462
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,4,128,1,float16,float16,0,1.2745386759440105
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,48,8,128,1,float16,float16,0,1.2757333119710286
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,1,128,1,float16,float16,0,11.21655527750651
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,2,128,1,float16,float16,0,11.47869873046875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,48,128,1,float16,float16,0,12.488703409830729
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,48,2,128,1,float16,float16,0,22.592852274576824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,48,1,128,1,float16,float16,0,22.610944112141926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,48,128,1,float16,float16,0,6.0967254638671875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,48,4,128,1,float16,float16,0,23.31323750813802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,48,8,128,1,float16,float16,0,23.39874013264974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,1,128,1,float16,float16,0,5.350912094116211
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,1,128,1,float16,float16,0,2.7567787170410156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,48,128,1,float16,float16,0,3.030869483947754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,2,128,1,float16,float16,0,5.480277379353841
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,2,128,1,float16,float16,0,2.7649707794189453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,4,128,1,float16,float16,0,5.661696116129558
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,8,128,1,float16,float16,0,11.552597045898438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,48,8,128,1,float16,float16,0,5.616128285725911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,48,4,128,1,float16,float16,0,11.453951517740885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,4,128,1,float16,float16,0,2.8144639333089194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,48,128,1,float16,float16,0,1.543168067932129
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,2,128,1,float16,float16,0,1.4187520345052083
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,1,128,1,float16,float16,0,1.3784747123718262
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,4,128,1,float16,float16,0,1.4580052693684895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,48,8,128,1,float16,float16,0,2.77128537495931
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,48,8,128,1,float16,float16,0,1.4399147033691406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,1,128,1,float16,float16,0,0.7137280305226644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,2,128,1,float16,float16,0,0.7132159868876139
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,4,128,1,float16,float16,0,0.7357439994812012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,8,128,1,float16,float16,0,0.7456426620483398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,48,48,128,1,float16,float16,0,0.7999146779378256
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,1,128,1,float16,float16,0,10.686805725097656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,4,128,1,float16,float16,0,10.82419204711914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,48,128,1,float16,float16,0,12.06988779703776
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,2,128,1,float16,float16,0,10.680831909179688
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,1,128,1,float16,float16,0,4.949674606323242
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,48,4,128,1,float16,float16,0,22.38532257080078
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,2,128,1,float16,float16,0,5.084159851074219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,48,128,1,float16,float16,0,5.895338694254558
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,48,1,128,1,float16,float16,0,21.30141830444336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,48,8,128,1,float16,float16,0,22.23547617594401
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,1,128,1,float16,float16,0,2.4862720171610513
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,48,8,128,1,float16,float16,0,11.054762522379557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,48,128,1,float16,float16,0,2.939903895060221
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,4,128,1,float16,float16,0,2.6975574493408203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,2,128,1,float16,float16,0,2.5886720021565757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,48,2,128,1,float16,float16,0,21.29749298095703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,48,128,1,float16,float16,0,1.5653546651204426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,4,128,1,float16,float16,0,5.288618723551433
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,48,8,128,1,float16,float16,0,5.448703765869141
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,1,128,1,float16,float16,0,1.2569599946339924
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,48,8,128,1,float16,float16,0,2.655402660369873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,48,128,1,float16,float16,0,0.7965013186136881
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,2,128,1,float16,float16,0,1.2950186729431152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,4,128,1,float16,float16,0,1.33188263575236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,1,128,1,float16,float16,0,0.6698666413625082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,2,128,1,float16,float16,0,0.6886400381724039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,48,128,1,float16,float16,0,0.4145493507385254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,4,128,1,float16,float16,0,0.6917119820912679
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,1,128,1,float16,float16,0,0.35020800431569415
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,48,8,128,1,float16,float16,0,0.6995626290639242
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,48,8,128,1,float16,float16,0,1.3760852813720703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,2,128,1,float16,float16,0,0.355840007464091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,4,128,1,float16,float16,0,0.3804159959157308
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,48,8,128,1,float16,float16,0,0.3829760154088338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,1,128,1,float16,float16,0,6.052864074707031
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,4,128,1,float16,float16,0,6.584831873575847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,48,128,1,float16,float16,0,7.467007954915364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,2,128,1,float16,float16,0,6.079488118489583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,48,128,1,float16,float16,0,3.6572160720825195
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,1,128,1,float16,float16,0,2.9631147384643555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,48,1,128,1,float16,float16,0,12.733268737792969
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,2,128,1,float16,float16,0,3.0365012486775718
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,48,4,128,1,float16,float16,0,13.220181783040365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,48,8,128,1,float16,float16,0,13.294420878092447
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,48,8,128,1,float16,float16,0,6.558549245198567
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,1,128,1,float16,float16,0,1.490773359934489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,4,128,1,float16,float16,0,1.6360106468200684
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,2,128,1,float16,float16,0,1.5232000350952148
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,8,128,1,float16,float16,0,1.6489814122517903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,48,48,128,1,float16,float16,0,1.8416639963785808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,4,128,1,float16,float16,0,3.169109344482422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,48,8,128,1,float16,float16,0,3.158357302347819
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,48,2,128,1,float16,float16,0,12.705791473388672
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,2,128,1,float16,float16,0,0.7758506933848063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,4,128,1,float16,float16,0,0.8306346734364828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,1,128,1,float16,float16,0,0.7589546839396158
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,8,128,1,float16,float16,0,0.8258559703826904
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,48,128,1,float16,float16,0,0.5056853294372559
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,1,128,1,float16,float16,0,0.3969706694285075
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,48,48,128,1,float16,float16,0,0.9359359741210938
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,8,128,1,float16,float16,0,0.43161598841349286
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,4,128,1,float16,float16,0,0.4242773453394572
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,1,128,1,float16,float16,0,0.2152106761932373
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,48,128,1,float16,float16,0,0.2616320053736369
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,2,128,1,float16,float16,0,0.22374399503072104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,8,128,1,float16,float16,0,0.23296000560124716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,48,4,128,1,float16,float16,0,0.23091200987497965
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,48,2,128,1,float16,float16,0,0.40806400775909424
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,1,128,1,float16,float16,0,5.81222407023112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,48,128,1,float16,float16,0,7.548245112101237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,2,128,1,float16,float16,0,6.178815841674805
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,48,1,128,1,float16,float16,0,12.289877573649088
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,48,8,128,1,float16,float16,0,13.650090535481771
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,48,2,128,1,float16,float16,0,12.665855407714844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,48,128,1,float16,float16,0,3.787775993347168
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,48,4,128,1,float16,float16,0,13.183488210042318
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,8,128,1,float16,float16,0,6.547797520955403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,1,128,1,float16,float16,0,2.8738559087117515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,48,128,1,float16,float16,0,1.9002025922139485
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,4,128,1,float16,float16,0,3.2027308146158853
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,2,128,1,float16,float16,0,2.9663572311401367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,48,4,128,1,float16,float16,0,6.448469161987305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,2,128,1,float16,float16,0,1.5010132789611816
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,4,128,1,float16,float16,0,1.5844693183898926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,1,128,1,float16,float16,0,1.4559574127197266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,48,8,128,1,float16,float16,0,1.5981225967407227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,4,128,1,float16,float16,0,0.8101546764373779
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,1,128,1,float16,float16,0,0.7429119745890299
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,48,128,1,float16,float16,0,0.9656319618225098
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,48,8,128,1,float16,float16,0,3.1559680302937827
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,8,128,1,float16,float16,0,0.8202239672342936
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,48,128,1,float16,float16,0,0.5046613216400146
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,1,128,1,float16,float16,0,0.37990399201711017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,2,128,1,float16,float16,0,0.3928746779759725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,48,2,128,1,float16,float16,0,0.7656106948852539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,8,128,1,float16,float16,0,0.42769066492716473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,48,4,128,1,float16,float16,0,0.4227413336435954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,1,128,1,float16,float16,0,0.2053119937578837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,4,128,1,float16,float16,0,0.22374399503072104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,1,128,1,float16,float16,0,0.1155413289864858
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,2,128,1,float16,float16,0,0.21026132504145303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,4,128,1,float16,float16,0,0.12407466769218445
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,8,128,1,float16,float16,0,0.22801067431767783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,2,128,1,float16,float16,0,0.11997866630554199
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,48,128,1,float16,float16,0,0.14830933014551798
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,48,8,128,1,float16,float16,0,0.12526933352152506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,48,48,128,1,float16,float16,0,0.2650453249613444
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,1,128,1,float16,float16,0,3.654997189839681
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,48,128,1,float16,float16,0,5.587968190511067
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,2,128,1,float16,float16,0,3.8012587229410806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,4,128,1,float16,float16,0,4.261034647623698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,48,1,128,1,float16,float16,0,7.869781494140625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,1,128,1,float16,float16,0,1.8365440368652344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,48,128,1,float16,float16,0,2.7849388122558594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,48,128,1,float16,float16,0,1.3863253593444824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,4,128,1,float16,float16,0,2.184874693552653
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,48,2,128,1,float16,float16,0,7.892138799031575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,48,4,128,1,float16,float16,0,8.691541035970053
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,2,128,1,float16,float16,0,1.9450880686442058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,1,128,1,float16,float16,0,0.9074347019195557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,48,128,1,float16,float16,0,0.6981973648071289
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,1,128,1,float16,float16,0,0.4633599917093913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,2,128,1,float16,float16,0,0.9659732977549235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,48,8,128,1,float16,float16,0,2.1411840120951333
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,4,128,1,float16,float16,0,1.077077309290568
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,2,128,1,float16,float16,0,0.48878931999206543
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,4,128,1,float16,float16,0,0.5406719843546549
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,48,8,128,1,float16,float16,0,0.5483520030975342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,2,128,1,float16,float16,0,0.25565866629282635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,4,128,1,float16,float16,0,0.28125866254170734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,48,8,128,1,float16,float16,0,4.301141421000163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,8,128,1,float16,float16,0,0.2851840058962504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,1,128,1,float16,float16,0,0.24285866816838583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,48,128,1,float16,float16,0,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,1,128,1,float16,float16,0,0.1327786644299825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,8,128,1,float16,float16,0,0.1527466674645742
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,4,128,1,float16,float16,0,0.15035733580589294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,48,8,128,1,float16,float16,0,1.0745173295338948
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,4,128,1,float16,float16,0,0.08260266482830048
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,1,128,1,float16,float16,0,0.07560533285140991
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,48,128,1,float16,float16,0,0.106495996316274
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,2,128,1,float16,float16,0,0.0769706666469574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,48,48,128,1,float16,float16,0,0.3604480028152466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,48,128,1,float16,float16,0,0.06382933259010315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,1,128,1,float16,float16,0,0.04539733131726583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,4,128,1,float16,float16,0,0.05017599960168203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,2,128,1,float16,float16,0,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,48,8,128,1,float16,float16,0,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,48,8,128,1,float16,float16,0,0.08567466338475545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,48,2,128,1,float16,float16,0,0.13994666934013367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,1,128,1,float16,float16,0,1.2929706573486328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,48,128,1,float16,float16,0,2.2712319691975913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,48,8,128,1,float16,float16,0,8.831829071044922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,2,128,1,float16,float16,0,1.4004906018575032
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,48,1,128,1,float16,float16,0,2.6291200319925943
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,1,128,1,float16,float16,0,0.6352213223775228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,48,2,128,1,float16,float16,0,2.791253407796224
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,48,4,128,1,float16,float16,0,3.225599924723307
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,4,128,1,float16,float16,0,1.5982933044433594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,2,128,1,float16,float16,0,0.6939307053883871
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,1,128,1,float16,float16,0,0.3278506596883138
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,8,128,1,float16,float16,0,0.8046933015187582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,4,128,1,float16,float16,0,0.7949653466542562
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,48,128,1,float16,float16,0,0.5713920195897421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,2,128,1,float16,float16,0,0.35498666763305664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,48,128,1,float16,float16,0,0.29474133253097534
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,8,128,1,float16,float16,0,0.4130133390426636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,1,128,1,float16,float16,0,0.17169066270192465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,48,4,128,1,float16,float16,0,0.40243200461069745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,48,8,128,1,float16,float16,0,1.6261119842529297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,48,48,128,1,float16,float16,0,1.1267413298288982
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,2,128,1,float16,float16,0,0.18397865692774454
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,4,128,1,float16,float16,0,0.20940800507863364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,48,128,1,float16,float16,0,0.15479466319084167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,48,8,128,1,float16,float16,0,0.21333332856496176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,4,128,1,float16,float16,0,0.11281067132949829
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,2,128,1,float16,float16,0,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,8,128,1,float16,float16,0,0.11451733112335205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,48,128,1,float16,float16,0,0.08567466338475545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,1,128,1,float16,float16,0,0.053077335158983864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,48,128,1,float16,float16,0,0.048810665806134544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,4,128,1,float16,float16,0,0.06297599772612254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,8,128,1,float16,float16,0,0.06400000055631001
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,48,2,128,1,float16,float16,0,0.05614933371543884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,1,128,1,float16,float16,0,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,48,1,128,1,float16,float16,0,0.09471999605496724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,48,8,128,1,float16,float16,0,3.261098543802897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,4,128,1,float16,float16,0,0.03566933423280716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,1,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,8,128,1,float16,float16,0,0.03583999971548716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,2,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,48,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,48,2,128,1,float16,float16,0,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,8,128,1,float16,float16,0,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,48,4,128,1,float16,float16,0,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,1,128,1,float16,float16,0,0.4928853511810303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,48,128,1,float16,float16,0,1.1019946734110515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,48,1,128,1,float16,float16,0,1.1535360018412273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,48,2,128,1,float16,float16,0,1.2648106416066487
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,2,128,1,float16,float16,0,0.5476693312327067
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,4,128,1,float16,float16,0,0.6584320068359375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,48,4,128,1,float16,float16,0,1.509717305501302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,48,128,1,float16,float16,0,0.5367466608683268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,1,128,1,float16,float16,0,0.2500266631444295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,2,128,1,float16,float16,0,0.27716267108917236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,48,8,128,1,float16,float16,0,1.5356586774190266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,4,128,1,float16,float16,0,0.33211733897527057
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,1,128,1,float16,float16,0,0.13243732849756876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,2,128,1,float16,float16,0,0.14574933052062988
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,48,8,128,1,float16,float16,0,0.34065067768096924
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,48,8,128,1,float16,float16,0,0.6758399804433187
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,1,128,1,float16,float16,0,0.07423999905586243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,2,128,1,float16,float16,0,0.08004266520341237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,48,128,1,float16,float16,0,0.1508693297704061
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,48,128,1,float16,float16,0,0.2826240062713623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,48,128,1,float16,float16,0,0.08004266520341237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,1,128,1,float16,float16,0,0.04181333382924398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,4,128,1,float16,float16,0,0.09233066439628601
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,2,128,1,float16,float16,0,0.04505600035190582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,8,128,1,float16,float16,0,0.17578667402267456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,4,128,1,float16,float16,0,0.05034666756788889
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,48,128,1,float16,float16,0,0.04420266548792521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,1,128,1,float16,float16,0,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,48,8,128,1,float16,float16,0,0.05444266895453135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,2,128,1,float16,float16,0,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,48,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,1,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,8,128,1,float16,float16,0,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,48,8,128,1,float16,float16,0,0.09591466188430786
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,4,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,2,128,1,float16,float16,0,0.01757866640885671
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,48,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,48,4,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,48,4,128,1,float16,float16,0,0.17100799083709717
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,1,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,4,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,8,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,48,8,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,48,2,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,2,128,1,float16,float16,0,0.2727253238360087
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,1,128,1,float16,float16,0,0.24576000372568765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,4,128,1,float16,float16,0,0.32614399989446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,48,1,128,1,float16,float16,0,0.4742826620737712
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,8,128,1,float16,float16,0,0.3346773386001587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,1,128,1,float16,float16,0,0.13363200426101685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,48,4,128,1,float16,float16,0,0.6372693379720052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,48,128,1,float16,float16,0,0.28330665826797485
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,8,128,1,float16,float16,0,0.17493333419164023
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,4,128,1,float16,float16,0,0.17117865880330405
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,48,2,128,1,float16,float16,0,0.14506666858990988
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,48,2,128,1,float16,float16,0,0.5309439897537231
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,48,8,128,1,float16,float16,0,0.6642346779505411
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,2,128,1,float16,float16,0,0.07987200220425923
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,4,128,1,float16,float16,0,0.09284266829490662
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,2,128,1,float16,float16,0,0.04454400142033895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,8,128,1,float16,float16,0,0.09454933802286784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,48,128,1,float16,float16,0,0.14643200238545737
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,4,128,1,float16,float16,0,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,8,128,1,float16,float16,0,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,48,128,1,float16,float16,0,0.04334933559099833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,48,1,128,1,float16,float16,0,0.07526400188604991
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,2,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,48,128,1,float16,float16,0,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,4,128,1,float16,float16,0,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,8,128,1,float16,float16,0,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,48,48,128,1,float16,float16,0,0.5360639890034994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,48,1,128,1,float16,float16,0,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,2,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,4,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,1,128,1,float16,float16,0,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,48,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,4,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,2,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,48,8,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,48,128,1,float16,float16,0,0.07884799937407176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,48,1,128,1,float16,float16,0,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,4,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,48,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,48,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,48,8,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,1,128,1,float16,float16,0,0.23500800132751465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,48,1,128,1,float16,float16,0,0.4729173183441162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,48,2,128,1,float16,float16,0,0.4667733510335286
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,48,4,128,1,float16,float16,0,0.5154133240381876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,48,128,1,float16,float16,0,0.3449173370997111
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,48,128,1,float16,float16,0,0.1781760056813558
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,4,128,1,float16,float16,0,0.26368000109990436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,2,128,1,float16,float16,0,0.24200532833735147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,48,8,128,1,float16,float16,0,0.5198506514231364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,48,8,128,1,float16,float16,0,0.26658133665720624
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,48,128,1,float16,float16,0,0.0962559978167216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,8,128,1,float16,float16,0,0.14114133516947427
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,4,128,1,float16,float16,0,0.13960533340771994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,2,128,1,float16,float16,0,0.06656000018119812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,1,128,1,float16,float16,0,0.12236799796422322
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,48,128,1,float16,float16,0,0.053247998158137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,1,128,1,float16,float16,0,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,8,128,1,float16,float16,0,0.07628799974918365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,4,128,1,float16,float16,0,0.0747519979874293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,2,128,1,float16,float16,0,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,8,128,1,float16,float16,0,0.03857066730658213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,48,4,128,1,float16,float16,0,0.03839999934037527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,48,1,128,1,float16,float16,0,0.06400000055631001
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,48,2,128,1,float16,float16,0,0.12748799721399942
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,48,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,8,128,1,float16,float16,0,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,2,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,48,128,1,float16,float16,0,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,4,128,1,float16,float16,0,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,48,1,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,1,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,8,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,2,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,8,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,48,4,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,48,48,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,48,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,48,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,1,128,1,float16,float16,0,0.22050132354100546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,4,128,1,float16,float16,0,0.2373973329861959
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,2,128,1,float16,float16,0,0.2259626587231954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,48,128,1,float16,float16,0,0.2728959918022156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,48,8,128,1,float16,float16,0,0.46882132689158124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,48,4,128,1,float16,float16,0,0.4659200112024943
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,48,2,128,1,float16,float16,0,0.44287999471028644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,48,128,1,float16,float16,0,0.1430186629295349
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,48,1,128,1,float16,float16,0,0.4312746524810791
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,1,128,1,float16,float16,0,0.11400533715883891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,48,8,128,1,float16,float16,0,0.2397866646448771
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,2,128,1,float16,float16,0,0.11741866668065389
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,48,128,1,float16,float16,0,0.07816533247629802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,8,128,1,float16,float16,0,0.12424533565839131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,2,128,1,float16,float16,0,0.059562668204307556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,48,4,128,1,float16,float16,0,0.12066133817036946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,48,128,1,float16,float16,0,0.03925333420435587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,8,128,1,float16,float16,0,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,1,128,1,float16,float16,0,0.05819733440876007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,48,4,128,1,float16,float16,0,0.06178133189678192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,4,128,1,float16,float16,0,0.03618133316437403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,48,128,1,float16,float16,0,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,2,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,1,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,8,128,1,float16,float16,0,0.03618133316437403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,2,128,1,float16,float16,0,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,48,1,128,1,float16,float16,0,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,4,128,1,float16,float16,0,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,8,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,48,128,1,float16,float16,0,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,48,8,128,1,float16,float16,0,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,4,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,1,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,48,2,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,48,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,48,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,48,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,48,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,0,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,0,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,48,128,1,float16,float16,0,0.19165867567062378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,0,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,0,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,0,0.09284266829490662
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,48,128,1,float16,float16,0,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,0,0.03822933385769526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,0,0.054101333022117615
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,0,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,48,128,1,float16,float16,0,0.056320001681645714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,0,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,0,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,0,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,48,128,1,float16,float16,0,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,0,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,48,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,48,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,48,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,48,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,1,128,1,float16,float16,0,60.30882263183594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,2,128,1,float16,float16,0,60.71569315592448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,40,128,1,float16,float16,0,61.4469960530599
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,4,128,1,float16,float16,0,61.78201802571615
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,1,128,1,float16,float16,0,29.94244384765625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,40,128,1,float16,float16,0,31.25794219970703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,2,128,1,float16,float16,0,30.730580647786457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,40,128,1,float16,float16,0,15.746560414632162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,40,4,128,1,float16,float16,0,120.44698079427083
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,40,1,128,1,float16,float16,0,120.45635986328125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,40,8,128,1,float16,float16,0,61.478912353515625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,4,128,1,float16,float16,0,30.21107228597005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,40,2,128,1,float16,float16,0,121.24381510416667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,40,8,128,1,float16,float16,0,121.91436767578125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,40,8,128,1,float16,float16,0,30.76215362548828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,1,128,1,float16,float16,0,15.290026346842447
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,2,128,1,float16,float16,0,15.06372324625651
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,4,128,1,float16,float16,0,15.27347183227539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,40,8,128,1,float16,float16,0,15.490559895833334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,1,128,1,float16,float16,0,33.6535898844401
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,40,128,1,float16,float16,0,35.95008087158203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,2,128,1,float16,float16,0,34.06284840901693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,4,128,1,float16,float16,0,34.98939768473307
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,1,128,1,float16,float16,0,17.03918965657552
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,40,128,1,float16,float16,0,18.037760416666668
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,40,1,128,1,float16,float16,0,68.10743713378906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,2,128,1,float16,float16,0,17.289386749267578
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,40,2,128,1,float16,float16,0,68.5648193359375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,40,4,128,1,float16,float16,0,69.39955139160156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,40,8,128,1,float16,float16,0,69.70999654134114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,40,128,1,float16,float16,0,8.829952239990234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,1,128,1,float16,float16,0,8.642218907674154
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,2,128,1,float16,float16,0,8.556714375813803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,4,128,1,float16,float16,0,8.677034378051758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,40,8,128,1,float16,float16,0,8.738815943400065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,40,8,128,1,float16,float16,0,34.36151377360026
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,4,128,1,float16,float16,0,17.549994150797527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,40,8,128,1,float16,float16,0,17.498624165852863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,40,128,1,float16,float16,0,25.19500732421875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,1,128,1,float16,float16,0,23.5142822265625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,2,128,1,float16,float16,0,24.042325337727863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,4,128,1,float16,float16,0,24.688980102539062
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,40,128,1,float16,float16,0,12.340906778971354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,1,128,1,float16,float16,0,12.02517318725586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,2,128,1,float16,float16,0,12.186965942382812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,40,1,128,1,float16,float16,0,47.39293924967448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,40,2,128,1,float16,float16,0,48.48913065592448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,40,4,128,1,float16,float16,0,48.6461435953776
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,40,8,128,1,float16,float16,0,48.71065775553385
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,40,8,128,1,float16,float16,0,24.510635375976562
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,4,128,1,float16,float16,0,12.036436716715494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,1,128,1,float16,float16,0,6.019754409790039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,40,128,1,float16,float16,0,6.171648025512695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,2,128,1,float16,float16,0,5.961557388305664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,4,128,1,float16,float16,0,6.059520085652669
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,40,8,128,1,float16,float16,0,12.318548838297525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,40,8,128,1,float16,float16,0,6.002005259195964
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,1,128,1,float16,float16,0,31.300608317057293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,40,128,1,float16,float16,0,32.63778177897135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,4,128,1,float16,float16,0,31.874900817871094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,2,128,1,float16,float16,0,30.9212163289388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,40,128,1,float16,float16,0,16.364373524983723
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,1,128,1,float16,float16,0,15.509845733642578
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,2,128,1,float16,float16,0,15.504554748535156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,40,1,128,1,float16,float16,0,61.31934102376302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,40,2,128,1,float16,float16,0,62.49898783365885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,40,4,128,1,float16,float16,0,62.215850830078125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,40,128,1,float16,float16,0,7.991125106811523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,40,8,128,1,float16,float16,0,63.069183349609375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,40,8,128,1,float16,float16,0,31.969622294108074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,1,128,1,float16,float16,0,7.693824132283528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,8,128,1,float16,float16,0,15.887359619140625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,40,4,128,1,float16,float16,0,15.83086903889974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,40,128,1,float16,float16,0,4.001962661743164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,2,128,1,float16,float16,0,7.824384053548177
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,4,128,1,float16,float16,0,7.725738525390625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,40,8,128,1,float16,float16,0,7.809535980224609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,1,128,1,float16,float16,0,3.816277186075846
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,2,128,1,float16,float16,0,3.8657706578572593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,4,128,1,float16,float16,0,4.012373288472493
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,40,8,128,1,float16,float16,0,3.912362734476725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,1,128,1,float16,float16,0,17.544362386067707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,2,128,1,float16,float16,0,17.971200307210285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,4,128,1,float16,float16,0,18.125823974609375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,40,128,1,float16,float16,0,18.972501118977863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,40,128,1,float16,float16,0,9.38427734375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,1,128,1,float16,float16,0,8.766976038614908
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,2,128,1,float16,float16,0,8.775168100992838
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,40,1,128,1,float16,float16,0,35.272875467936196
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,40,2,128,1,float16,float16,0,36.04070281982422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,40,4,128,1,float16,float16,0,35.805013020833336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,40,8,128,1,float16,float16,0,36.10419209798177
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,40,8,128,1,float16,float16,0,17.910442352294922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,40,128,1,float16,float16,0,4.684970537821452
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,4,128,1,float16,float16,0,8.995328267415365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,40,8,128,1,float16,float16,0,9.007957458496094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,40,128,1,float16,float16,0,2.371071974436442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,2,128,1,float16,float16,0,4.423850695292155
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,1,128,1,float16,float16,0,4.254549344380696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,4,128,1,float16,float16,0,4.417365392049153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,40,8,128,1,float16,float16,0,4.433919906616211
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,1,128,1,float16,float16,0,2.203477382659912
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,2,128,1,float16,float16,0,2.2814720471700034
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,8,128,1,float16,float16,0,2.249728043874105
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,40,4,128,1,float16,float16,0,2.2596267064412436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,2,128,1,float16,float16,0,16.255828857421875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,1,128,1,float16,float16,0,16.326314290364582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,4,128,1,float16,float16,0,16.8635737101237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,40,128,1,float16,float16,0,17.83739725748698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,1,128,1,float16,float16,0,7.940437316894531
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,40,128,1,float16,float16,0,8.745471954345703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,40,1,128,1,float16,float16,0,32.58880106608073
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,40,4,128,1,float16,float16,0,33.129130045572914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,40,2,128,1,float16,float16,0,32.645121256510414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,40,8,128,1,float16,float16,0,33.648640950520836
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,2,128,1,float16,float16,0,7.9919789632161455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,40,8,128,1,float16,float16,0,16.6647466023763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,4,128,1,float16,float16,0,8.20957883199056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,1,128,1,float16,float16,0,3.8988800048828125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,40,8,128,1,float16,float16,0,8.389461517333984
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,40,128,1,float16,float16,0,4.303189277648926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,4,128,1,float16,float16,0,4.046165466308594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,2,128,1,float16,float16,0,3.9273811976114907
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,40,128,1,float16,float16,0,2.2227627436319985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,1,128,1,float16,float16,0,2.003455956776937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,2,128,1,float16,float16,0,1.9938987096150715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,40,128,1,float16,float16,0,1.1279359658559163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,4,128,1,float16,float16,0,2.0882773399353027
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,2,128,1,float16,float16,0,1.0424319903055828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,40,8,128,1,float16,float16,0,2.0640427271525064
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,1,128,1,float16,float16,0,1.0379947026570637
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,40,8,128,1,float16,float16,0,4.078762690226237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,4,128,1,float16,float16,0,1.0671786467234294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,40,8,128,1,float16,float16,0,1.0588160355885823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,2,128,1,float16,float16,0,9.47319475809733
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,4,128,1,float16,float16,0,9.652565638224283
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,1,128,1,float16,float16,0,9.410730361938477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,40,128,1,float16,float16,0,10.371072133382162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,1,128,1,float16,float16,0,4.614485422770183
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,40,128,1,float16,float16,0,5.098154703776042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,40,1,128,1,float16,float16,0,18.663424173990887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,40,2,128,1,float16,float16,0,19.19146728515625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,40,4,128,1,float16,float16,0,19.238229115804035
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,40,8,128,1,float16,float16,0,19.844437917073567
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,40,8,128,1,float16,float16,0,9.839957555135092
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,2,128,1,float16,float16,0,4.5446828206380205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,2,128,1,float16,float16,0,2.324138641357422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,1,128,1,float16,float16,0,2.29529603322347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,4,128,1,float16,float16,0,4.748799959818522
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,40,8,128,1,float16,float16,0,4.68940798441569
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,40,128,1,float16,float16,0,2.5535146395365396
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,4,128,1,float16,float16,0,2.445312023162842
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,40,8,128,1,float16,float16,0,2.3736319541931152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,40,128,1,float16,float16,0,1.31822935740153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,1,128,1,float16,float16,0,1.157802661259969
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,4,128,1,float16,float16,0,1.222656011581421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,2,128,1,float16,float16,0,1.234602689743042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,40,128,1,float16,float16,0,0.695466677347819
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,40,8,128,1,float16,float16,0,1.2161706288655598
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,2,128,1,float16,float16,0,0.6220800081888834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,1,128,1,float16,float16,0,0.6155946652094523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,4,128,1,float16,float16,0,0.6372693379720052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,40,8,128,1,float16,float16,0,0.6435840129852295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,1,128,1,float16,float16,0,8.779946645100912
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,2,128,1,float16,float16,0,8.854186375935873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,40,128,1,float16,float16,0,4.93448543548584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,40,8,128,1,float16,float16,0,18.68014907836914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,40,4,128,1,float16,float16,0,18.700458526611328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,4,128,1,float16,float16,0,9.225215911865234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,40,128,1,float16,float16,0,10.131455739339193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,40,1,128,1,float16,float16,0,17.873578389485676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,40,2,128,1,float16,float16,0,18.124117533365887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,2,128,1,float16,float16,0,4.302677472432454
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,1,128,1,float16,float16,0,4.111701329549153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,4,128,1,float16,float16,0,4.5335893630981445
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,40,8,128,1,float16,float16,0,9.37710952758789
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,40,8,128,1,float16,float16,0,4.52625052134196
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,40,128,1,float16,float16,0,1.2861440181732178
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,40,128,1,float16,float16,0,2.491391976674398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,8,128,1,float16,float16,0,2.2830079396565757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,4,128,1,float16,float16,0,2.2519466082255044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,1,128,1,float16,float16,0,2.1113173166910806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,1,128,1,float16,float16,0,1.0752000013987224
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,40,2,128,1,float16,float16,0,2.116096019744873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,1,128,1,float16,float16,0,0.5521066586176554
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,40,128,1,float16,float16,0,0.6579200029373169
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,2,128,1,float16,float16,0,1.1086506843566895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,4,128,1,float16,float16,0,1.140565315882365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,2,128,1,float16,float16,0,0.5725866556167603
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,4,128,1,float16,float16,0,0.5899946689605713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,40,8,128,1,float16,float16,0,0.5927253166834513
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,40,128,1,float16,float16,0,0.3515733480453491
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,40,8,128,1,float16,float16,0,1.147221326828003
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,1,128,1,float16,float16,0,0.3007146716117859
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,2,128,1,float16,float16,0,0.3131733338038127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,4,128,1,float16,float16,0,0.31948800881703693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,40,8,128,1,float16,float16,0,0.3227306604385376
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,4,128,1,float16,float16,0,5.471914927164714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,1,128,1,float16,float16,0,5.022378603617351
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,40,128,1,float16,float16,0,6.148949305216472
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,2,128,1,float16,float16,0,5.1249494552612305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,40,4,128,1,float16,float16,0,11.073365529378256
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,40,8,128,1,float16,float16,0,11.284138997395834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,40,128,1,float16,float16,0,3.0556160608927407
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,1,128,1,float16,float16,0,2.4535039265950522
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,40,2,128,1,float16,float16,0,10.68014907836914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,2,128,1,float16,float16,0,2.573311964670817
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,40,1,128,1,float16,float16,0,10.481322606404623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,40,8,128,1,float16,float16,0,5.5965010325113935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,40,128,1,float16,float16,0,1.5486292839050293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,1,128,1,float16,float16,0,1.2325546741485596
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,4,128,1,float16,float16,0,2.7106987635294595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,40,8,128,1,float16,float16,0,2.7089920043945312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,4,128,1,float16,float16,0,1.3672107060750325
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,40,128,1,float16,float16,0,0.8110079765319824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,2,128,1,float16,float16,0,1.2887039979298909
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,2,128,1,float16,float16,0,0.6568959951400757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,4,128,1,float16,float16,0,0.6956373055775961
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,1,128,1,float16,float16,0,0.6364159981409708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,40,128,1,float16,float16,0,0.4208639860153198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,40,8,128,1,float16,float16,0,0.7135573228200277
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,2,128,1,float16,float16,0,0.34867199261983234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,1,128,1,float16,float16,0,0.3418453137079875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,40,8,128,1,float16,float16,0,1.3806932767232258
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,8,128,1,float16,float16,0,0.3694933255513509
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,40,4,128,1,float16,float16,0,0.36693334579467773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,4,128,1,float16,float16,0,0.2032639980316162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,40,128,1,float16,float16,0,0.22920533021291098
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,1,128,1,float16,float16,0,0.18943999210993448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,2,128,1,float16,float16,0,0.19541333119074503
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,40,8,128,1,float16,float16,0,0.20480000972747803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,1,128,1,float16,float16,0,4.86024538675944
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,2,128,1,float16,float16,0,5.086378733317058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,40,128,1,float16,float16,0,6.466901143391927
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,4,128,1,float16,float16,0,5.480447769165039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,40,8,128,1,float16,float16,0,11.400362650553385
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,1,128,1,float16,float16,0,2.389845371246338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,40,2,128,1,float16,float16,0,9.938090642293295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,40,1,128,1,float16,float16,0,10.227882385253906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,40,128,1,float16,float16,0,3.179178555806478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,40,8,128,1,float16,float16,0,5.482666651407878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,2,128,1,float16,float16,0,2.509653409322103
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,2,128,1,float16,float16,0,1.2511573632558186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,1,128,1,float16,float16,0,1.226581335067749
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,8,128,1,float16,float16,0,2.7281068166097007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,40,4,128,1,float16,float16,0,2.721280097961426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,40,128,1,float16,float16,0,1.6092160542805989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,8,128,1,float16,float16,0,1.3998079299926758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,40,4,128,1,float16,float16,0,1.3697706858317058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,40,128,1,float16,float16,0,0.8192000389099121
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,1,128,1,float16,float16,0,0.6082559823989868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,2,128,1,float16,float16,0,0.641706665356954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,4,128,1,float16,float16,0,0.6937599976857504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,40,8,128,1,float16,float16,0,0.7012693087259928
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,40,128,1,float16,float16,0,0.4270079930623372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,4,128,1,float16,float16,0,0.35942399501800537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,1,128,1,float16,float16,0,0.31948800881703693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,2,128,1,float16,float16,0,0.3333119948705037
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,40,128,1,float16,float16,0,0.2295466661453247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,40,8,128,1,float16,float16,0,0.36471466223398846
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,2,128,1,float16,float16,0,0.183296004931132
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,4,128,1,float16,float16,0,0.19404800732930502
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,1,128,1,float16,float16,0,0.17612799008687338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,1,128,1,float16,float16,0,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,40,128,1,float16,float16,0,0.13141333063443503
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,40,8,128,1,float16,float16,0,0.19746132691701254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,40,4,128,1,float16,float16,0,11.109546661376953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,2,128,1,float16,float16,0,0.1013759970664978
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,4,128,1,float16,float16,0,0.1083733340104421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,40,8,128,1,float16,float16,0,0.11229866743087769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,40,128,1,float16,float16,0,4.614826520284017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,1,128,1,float16,float16,0,3.06551456451416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,2,128,1,float16,float16,0,3.215189297993978
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,40,1,128,1,float16,float16,0,6.2890669504801435
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,40,128,1,float16,float16,0,2.3381333351135254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,1,128,1,float16,float16,0,1.5392427444458008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,4,128,1,float16,float16,0,3.654143969217936
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,40,2,128,1,float16,float16,0,6.752085367838542
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,40,8,128,1,float16,float16,0,3.7044906616210938
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,2,128,1,float16,float16,0,1.6544426282246907
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,40,128,1,float16,float16,0,1.1717973550160725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,4,128,1,float16,float16,0,0.9284266630808512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,1,128,1,float16,float16,0,0.777557373046875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,8,128,1,float16,float16,0,1.8638505935668945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,40,4,128,1,float16,float16,0,1.8368852933247883
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,40,8,128,1,float16,float16,0,7.571968078613281
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,1,128,1,float16,float16,0,0.39509332180023193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,2,128,1,float16,float16,0,0.822101354598999
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,40,128,1,float16,float16,0,0.5973333517710367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,2,128,1,float16,float16,0,0.22220800320307413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,40,4,128,1,float16,float16,0,7.492607752482097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,1,128,1,float16,float16,0,0.20974934101104736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,4,128,1,float16,float16,0,0.4747946659723918
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,8,128,1,float16,float16,0,0.48128000895182294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,40,8,128,1,float16,float16,0,0.9255253473917643
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,4,128,1,float16,float16,0,0.24473599592844644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,40,128,1,float16,float16,0,0.1699840029080709
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,8,128,1,float16,float16,0,0.1346560021241506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,40,2,128,1,float16,float16,0,0.42018131415049237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,1,128,1,float16,float16,0,0.11571199695269267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,40,128,1,float16,float16,0,0.09608532985051473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,2,128,1,float16,float16,0,0.12134400010108948
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,1,128,1,float16,float16,0,0.06417066852251689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,2,128,1,float16,float16,0,0.06809600194295247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,4,128,1,float16,float16,0,0.07389866809050243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,1,128,1,float16,float16,0,0.04266666869322459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,2,128,1,float16,float16,0,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,40,128,1,float16,float16,0,0.057002668579419456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,40,8,128,1,float16,float16,0,0.07611733178297679
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,4,128,1,float16,float16,0,0.04642133414745331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,40,4,128,1,float16,float16,0,0.13312000036239624
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,40,8,128,1,float16,float16,0,0.04642133414745331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,8,128,1,float16,float16,0,0.252074658870697
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,40,128,1,float16,float16,0,1.8838186264038086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,1,128,1,float16,float16,0,1.0944853623708088
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,40,1,128,1,float16,float16,0,2.199893315633138
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,2,128,1,float16,float16,0,1.1975680192311604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,40,2,128,1,float16,float16,0,2.3876266479492188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,4,128,1,float16,float16,0,1.4008320172627766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,1,128,1,float16,float16,0,0.5364053249359131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,40,128,1,float16,float16,0,0.9524906476338705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,40,40,128,1,float16,float16,0,0.30856533845265705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,2,128,1,float16,float16,0,0.5915306806564331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,40,128,1,float16,float16,0,0.48418132464090985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,40,8,128,1,float16,float16,0,1.4272853533426921
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,40,4,128,1,float16,float16,0,2.813098589579264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,2,128,1,float16,float16,0,0.3025919993718465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,1,128,1,float16,float16,0,0.27665066719055176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,4,128,1,float16,float16,0,0.6937599976857504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,40,8,128,1,float16,float16,0,0.7069013118743896
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,1,128,1,float16,float16,0,0.14728533228238425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,2,128,1,float16,float16,0,0.1604266663392385
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,4,128,1,float16,float16,0,0.3577173153559367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,1,128,1,float16,float16,0,0.08260266482830048
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,40,128,1,float16,float16,0,0.1358506679534912
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,4,128,1,float16,float16,0,0.18517333269119263
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,2,128,1,float16,float16,0,0.08840533097585042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,4,128,1,float16,float16,0,0.10120532910029094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,8,128,1,float16,float16,0,0.18961066007614136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,40,8,128,1,float16,float16,0,0.3638613224029541
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,1,128,1,float16,float16,0,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,40,8,128,1,float16,float16,0,0.10291199882825215
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,40,128,1,float16,float16,0,0.07799466451009114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,2,128,1,float16,float16,0,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,4,128,1,float16,float16,0,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,40,128,1,float16,float16,0,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,40,8,128,1,float16,float16,0,0.056661332647005715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,1,128,1,float16,float16,0,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,2,128,1,float16,float16,0,0.030378667016824085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,40,40,128,1,float16,float16,0,0.2537813385327657
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,4,128,1,float16,float16,0,0.03259733319282532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,40,8,128,1,float16,float16,0,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,1,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,40,128,1,float16,float16,0,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,2,128,1,float16,float16,0,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,8,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,40,4,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,40,8,128,1,float16,float16,0,2.8342612584431968
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,1,128,1,float16,float16,0,0.41574398676554364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,40,128,1,float16,float16,0,0.9110186894734701
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,40,1,128,1,float16,float16,0,0.989525318145752
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,40,4,128,1,float16,float16,0,1.291093349456787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,40,2,128,1,float16,float16,0,1.0774186452229817
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,4,128,1,float16,float16,0,0.5824853181838989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,40,128,1,float16,float16,0,0.4618240197499593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,2,128,1,float16,float16,0,0.47547733783721924
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,40,8,128,1,float16,float16,0,0.5983573198318481
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,1,128,1,float16,float16,0,0.21435733636220297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,2,128,1,float16,float16,0,0.23910399278004965
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,40,128,1,float16,float16,0,0.2384213407834371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,4,128,1,float16,float16,0,0.2949120004971822
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,2,128,1,float16,float16,0,0.1276586651802063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,40,8,128,1,float16,float16,0,1.3100372950236003
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,4,128,1,float16,float16,0,0.15308800339698792
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,1,128,1,float16,float16,0,0.11451733112335205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,1,128,1,float16,float16,0,0.06587733328342438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,40,8,128,1,float16,float16,0,0.3065173427263896
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,40,8,128,1,float16,float16,0,0.15820800264676413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,40,128,1,float16,float16,0,0.12868266304334006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,4,128,1,float16,float16,0,0.08516266942024231
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,40,128,1,float16,float16,0,0.07065600156784058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,8,128,1,float16,float16,0,0.08703999718030293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,2,128,1,float16,float16,0,0.03737599899371465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,40,128,1,float16,float16,0,0.037717332442601524
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,4,128,1,float16,float16,0,0.041984001795450844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,8,128,1,float16,float16,0,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,2,128,1,float16,float16,0,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,4,128,1,float16,float16,0,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,1,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,40,1,128,1,float16,float16,0,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,40,2,128,1,float16,float16,0,0.07116800049940745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,1,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,40,8,128,1,float16,float16,0,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,4,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,40,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,2,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,40,128,1,float16,float16,0,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,40,8,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,2,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,8,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,40,4,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,2,128,1,float16,float16,0,0.23910399278004965
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,1,128,1,float16,float16,0,0.20940800507863364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,4,128,1,float16,float16,0,0.28945066531499225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,40,128,1,float16,float16,0,0.2362026572227478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,8,128,1,float16,float16,0,0.2990079919497172
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,40,4,128,1,float16,float16,0,0.5655893484751383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,1,128,1,float16,float16,0,0.11468799908955891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,40,1,128,1,float16,float16,0,0.4055039882659912
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,4,128,1,float16,float16,0,0.1525759994983673
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,40,128,1,float16,float16,0,0.12834133704503378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,2,128,1,float16,float16,0,0.12782933314641318
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,40,2,128,1,float16,float16,0,0.4604586760203044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,40,8,128,1,float16,float16,0,0.15598932902018228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,40,40,128,1,float16,float16,0,0.45260798931121826
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,2,128,1,float16,float16,0,0.07099733253320058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,40,8,128,1,float16,float16,0,0.5855573415756226
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,8,128,1,float16,float16,0,0.0865280032157898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,4,128,1,float16,float16,0,0.040618665516376495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,1,128,1,float16,float16,0,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,4,128,1,float16,float16,0,0.08413867155710857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,1,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,40,128,1,float16,float16,0,0.03874133278926214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,2,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,8,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,40,1,128,1,float16,float16,0,0.06570666531721751
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,40,4,128,1,float16,float16,0,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,1,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,40,128,1,float16,float16,0,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,8,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,2,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,40,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,40,4,128,1,float16,float16,0,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,4,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,2,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,40,8,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,40,128,1,float16,float16,0,0.06980266670385997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,40,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,40,8,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,8,128,1,float16,float16,0,0.04420266548792521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,40,2,128,1,float16,float16,0,0.03498666733503342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,1,128,1,float16,float16,0,0.19421867529551187
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,2,128,1,float16,float16,0,0.20667733748753866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,40,2,128,1,float16,float16,0,0.400383989016215
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,40,1,128,1,float16,float16,0,0.37700267632802326
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,40,4,128,1,float16,float16,0,0.4474879900614421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,40,128,1,float16,float16,0,0.2943999965985616
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,4,128,1,float16,float16,0,0.22988800207773843
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,40,8,128,1,float16,float16,0,0.233130673567454
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,40,128,1,float16,float16,0,0.1546239952246348
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,40,8,128,1,float16,float16,0,0.45260798931121826
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,1,128,1,float16,float16,0,0.10376532872517903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,2,128,1,float16,float16,0,0.10854400197664897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,8,128,1,float16,float16,0,0.12270933389663696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,40,128,1,float16,float16,0,0.08396800359090169
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,2,128,1,float16,float16,0,0.05529599885145823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,1,128,1,float16,float16,0,0.051370665431022644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,8,128,1,float16,float16,0,0.06297599772612254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,40,128,1,float16,float16,0,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,2,128,1,float16,float16,0,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,1,128,1,float16,float16,0,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,4,128,1,float16,float16,0,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,40,8,128,1,float16,float16,0,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,40,4,128,1,float16,float16,0,0.12185600399971008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,40,4,128,1,float16,float16,0,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,40,128,1,float16,float16,0,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,1,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,2,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,4,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,40,8,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,8,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,40,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,40,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,1,128,1,float16,float16,0,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,2,128,1,float16,float16,0,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,40,4,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,2,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,4,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,40,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,40,40,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,2,128,1,float16,float16,0,0.1909760038057963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,1,128,1,float16,float16,0,0.18568533658981323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,4,128,1,float16,float16,0,0.20309333006540933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,40,1,128,1,float16,float16,0,0.36266668637593585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,40,128,1,float16,float16,0,0.23534933725992838
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,40,2,128,1,float16,float16,0,0.373418649037679
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,40,4,128,1,float16,float16,0,0.39714133739471436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,1,128,1,float16,float16,0,0.091648002465566
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,40,128,1,float16,float16,0,0.12390399972597758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,40,8,128,1,float16,float16,0,0.20565332969029745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,40,8,128,1,float16,float16,0,0.4010666608810425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,1,128,1,float16,float16,0,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,2,128,1,float16,float16,0,0.09301333626111348
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,40,128,1,float16,float16,0,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,40,128,1,float16,float16,0,0.06451199948787689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,4,128,1,float16,float16,0,0.05205333232879639
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,2,128,1,float16,float16,0,0.05017599960168203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,8,128,1,float16,float16,0,0.10376532872517903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,40,4,128,1,float16,float16,0,0.10171733299891154
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,40,8,128,1,float16,float16,0,0.05256533126036326
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,1,128,1,float16,float16,0,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,8,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,8,128,1,float16,float16,0,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,2,128,1,float16,float16,0,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,40,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,1,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,4,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,40,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,40,4,128,1,float16,float16,0,0.030378667016824085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,1,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,40,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,8,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,4,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,40,2,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,40,40,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,40,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,40,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,40,128,1,float16,float16,0,0.16179200013478598
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,0,0.03054933249950409
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,0,0.03737599899371465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,0,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,0,0.059903999169667564
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,0,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,0,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,40,128,1,float16,float16,0,0.08550399541854858
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,0,0.09045333663622539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,0,0.05341866612434387
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,40,128,1,float16,float16,0,0.04761599997679392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,0,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,0,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,0,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,0,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,0,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,40,128,1,float16,float16,0,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,0,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,40,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,40,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,40,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,40,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,2,128,1,float16,float16,0,47.68614196777344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,1,128,1,float16,float16,0,47.99573262532552
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,32,128,1,float16,float16,0,49.73840840657552
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,4,128,1,float16,float16,0,48.63829549153646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,32,128,1,float16,float16,0,24.809641520182293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,1,128,1,float16,float16,0,24.3770014444987
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,2,128,1,float16,float16,0,24.22869364420573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,32,8,128,1,float16,float16,0,48.26248677571615
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,4,128,1,float16,float16,0,24.45380147298177
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,32,1,128,1,float16,float16,0,96.49305216471355
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,32,4,128,1,float16,float16,0,96.5418701171875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,32,2,128,1,float16,float16,0,98.07411702473958
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,32,8,128,1,float16,float16,0,97.92819213867188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,32,8,128,1,float16,float16,0,24.459264119466145
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,32,128,1,float16,float16,0,12.441429138183594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,1,128,1,float16,float16,0,11.968170166015625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,2,128,1,float16,float16,0,12.200277964274088
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,4,128,1,float16,float16,0,12.328618367513021
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,32,8,128,1,float16,float16,0,12.220586140950521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,32,128,1,float16,float16,0,28.005035400390625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,1,128,1,float16,float16,0,27.233963012695312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,2,128,1,float16,float16,0,27.685887654622395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,4,128,1,float16,float16,0,27.40258026123047
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,32,128,1,float16,float16,0,14.220458984375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,2,128,1,float16,float16,0,13.626197814941406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,1,128,1,float16,float16,0,13.81051762898763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,32,1,128,1,float16,float16,0,55.44960021972656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,32,2,128,1,float16,float16,0,55.36375427246094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,32,4,128,1,float16,float16,0,56.050689697265625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,32,8,128,1,float16,float16,0,55.51786804199219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,32,128,1,float16,float16,0,7.147861480712891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,1,128,1,float16,float16,0,6.976853052775065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,2,128,1,float16,float16,0,6.944426854451497
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,4,128,1,float16,float16,0,6.991872151692708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,32,8,128,1,float16,float16,0,27.580245971679688
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,4,128,1,float16,float16,0,13.919743855794271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,32,8,128,1,float16,float16,0,6.990847905476888
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,32,8,128,1,float16,float16,0,13.797035217285156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,32,128,1,float16,float16,0,19.733503977457683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,1,128,1,float16,float16,0,19.076095581054688
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,2,128,1,float16,float16,0,19.371861775716145
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,4,128,1,float16,float16,0,19.32407506306966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,32,128,1,float16,float16,0,9.958741505940756
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,1,128,1,float16,float16,0,9.587882359822592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,2,128,1,float16,float16,0,9.545728047688803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,32,1,128,1,float16,float16,0,38.329002380371094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,32,2,128,1,float16,float16,0,39.30163319905599
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,32,4,128,1,float16,float16,0,39.79315185546875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,32,128,1,float16,float16,0,4.910421371459961
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,32,8,128,1,float16,float16,0,39.30180358886719
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,32,8,128,1,float16,float16,0,19.354624430338543
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,1,128,1,float16,float16,0,4.779007911682129
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,4,128,1,float16,float16,0,9.810944239298502
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,32,8,128,1,float16,float16,0,9.873749415079752
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,2,128,1,float16,float16,0,4.921855926513672
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,8,128,1,float16,float16,0,4.806997299194336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,32,4,128,1,float16,float16,0,4.969130516052246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,1,128,1,float16,float16,0,25.12964375813802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,2,128,1,float16,float16,0,24.707071940104168
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,32,128,1,float16,float16,0,25.796437581380207
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,4,128,1,float16,float16,0,25.476949055989582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,32,128,1,float16,float16,0,12.785663604736328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,1,128,1,float16,float16,0,12.375039418538412
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,2,128,1,float16,float16,0,12.565675099690756
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,32,1,128,1,float16,float16,0,49.54743448893229
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,32,2,128,1,float16,float16,0,49.512105305989586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,32,128,1,float16,float16,0,6.467072168986003
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,32,4,128,1,float16,float16,0,50.91532897949219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,32,8,128,1,float16,float16,0,50.83016459147135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,32,8,128,1,float16,float16,0,25.792340596516926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,1,128,1,float16,float16,0,6.140928268432617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,4,128,1,float16,float16,0,12.492287953694662
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,32,8,128,1,float16,float16,0,12.626261393229166
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,2,128,1,float16,float16,0,6.224725087483724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,4,128,1,float16,float16,0,6.3020375569661455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,32,128,1,float16,float16,0,3.315370559692383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,1,128,1,float16,float16,0,3.097599983215332
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,2,128,1,float16,float16,0,3.1266132990519204
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,32,8,128,1,float16,float16,0,6.3416322072347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,4,128,1,float16,float16,0,3.1926612854003906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,32,8,128,1,float16,float16,0,3.1778132120768228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,1,128,1,float16,float16,0,14.090240478515625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,2,128,1,float16,float16,0,14.359893798828125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,32,128,1,float16,float16,0,14.967466990152994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,32,128,1,float16,float16,0,7.533738454182942
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,1,128,1,float16,float16,0,6.893909454345703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,32,1,128,1,float16,float16,0,28.275711059570312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,32,2,128,1,float16,float16,0,29.15601094563802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,32,4,128,1,float16,float16,0,29.379071553548176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,32,8,128,1,float16,float16,0,28.95837910970052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,8,128,1,float16,float16,0,14.58517328898112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,32,4,128,1,float16,float16,0,14.361087799072266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,2,128,1,float16,float16,0,7.157418568929036
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,32,128,1,float16,float16,0,3.7447681427001953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,4,128,1,float16,float16,0,7.219541549682617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,1,128,1,float16,float16,0,3.4909868240356445
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,2,128,1,float16,float16,0,3.4909868240356445
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,32,128,1,float16,float16,0,1.919317404429118
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,4,128,1,float16,float16,0,3.573418617248535
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,1,128,1,float16,float16,0,1.8143572807312012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,4,128,1,float16,float16,0,1.8402986526489258
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,2,128,1,float16,float16,0,1.8343253135681152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,32,8,128,1,float16,float16,0,1.8705066045125325
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,32,8,128,1,float16,float16,0,7.236096064249675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,32,8,128,1,float16,float16,0,3.5406506856282554
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,1,128,1,float16,float16,0,12.784980773925781
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,2,128,1,float16,float16,0,13.193557739257812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,4,128,1,float16,float16,0,13.27786636352539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,32,128,1,float16,float16,0,14.077781677246094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,2,128,1,float16,float16,0,6.357333501180013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,32,128,1,float16,float16,0,6.872405370076497
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,1,128,1,float16,float16,0,6.221482594807942
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,32,1,128,1,float16,float16,0,26.1748046875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,32,2,128,1,float16,float16,0,26.290006001790363
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,32,8,128,1,float16,float16,0,27.272361755371094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,32,4,128,1,float16,float16,0,27.44439442952474
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,32,8,128,1,float16,float16,0,13.408426920572916
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,2,128,1,float16,float16,0,3.189759890238444
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,32,128,1,float16,float16,0,3.414698600769043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,1,128,1,float16,float16,0,3.163135846455892
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,4,128,1,float16,float16,0,6.63705571492513
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,32,128,1,float16,float16,0,1.798485279083252
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,32,8,128,1,float16,float16,0,6.735189437866211
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,4,128,1,float16,float16,0,3.376469294230143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,32,8,128,1,float16,float16,0,3.3464320500691733
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,1,128,1,float16,float16,0,1.616213321685791
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,32,128,1,float16,float16,0,0.9135786692301432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,1,128,1,float16,float16,0,0.8639146486918131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,2,128,1,float16,float16,0,1.6092160542805989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,4,128,1,float16,float16,0,1.677994728088379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,2,128,1,float16,float16,0,0.8702293237050375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,32,8,128,1,float16,float16,0,1.7034239768981934
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,4,128,1,float16,float16,0,0.8861013253529867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,32,8,128,1,float16,float16,0,0.884394645690918
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,2,128,1,float16,float16,0,7.720448176066081
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,32,128,1,float16,float16,0,8.355498631795248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,4,128,1,float16,float16,0,7.852885564168294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,1,128,1,float16,float16,0,7.384917577107747
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,1,128,1,float16,float16,0,3.650730768839518
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,32,128,1,float16,float16,0,4.085418701171875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,2,128,1,float16,float16,0,3.700906753540039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,32,2,128,1,float16,float16,0,15.288832346598307
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,32,1,128,1,float16,float16,0,15.000405629475912
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,32,4,128,1,float16,float16,0,15.690240224202475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,32,8,128,1,float16,float16,0,16.055124918619793
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,32,8,128,1,float16,float16,0,7.964330673217773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,32,128,1,float16,float16,0,2.0522665977478027
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,1,128,1,float16,float16,0,1.8488319714864094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,2,128,1,float16,float16,0,1.8795520464579265
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,4,128,1,float16,float16,0,3.843242645263672
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,32,8,128,1,float16,float16,0,3.8268585205078125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,32,128,1,float16,float16,0,1.087829351425171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,2,128,1,float16,float16,0,0.9716053009033203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,1,128,1,float16,float16,0,0.9504426320393881
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,4,128,1,float16,float16,0,0.9944746494293213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,8,128,1,float16,float16,0,1.9440639813741047
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,32,8,128,1,float16,float16,0,1.0156373182932537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,32,4,128,1,float16,float16,0,1.9367252985636394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,2,128,1,float16,float16,0,0.5137066841125488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,1,128,1,float16,float16,0,0.5111466646194458
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,32,128,1,float16,float16,0,0.5620053211847941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,4,128,1,float16,float16,0,0.5333333412806193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,32,8,128,1,float16,float16,0,0.5253119866053263
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,4,128,1,float16,float16,0,7.504213333129883
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,1,128,1,float16,float16,0,6.856362660725911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,32,128,1,float16,float16,0,8.258218765258789
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,2,128,1,float16,float16,0,7.063210805257161
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,32,128,1,float16,float16,0,3.970047950744629
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,32,1,128,1,float16,float16,0,14.252202351888021
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,1,128,1,float16,float16,0,3.37390931447347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,32,4,128,1,float16,float16,0,15.012522379557291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,32,2,128,1,float16,float16,0,14.418261210123697
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,32,8,128,1,float16,float16,0,15.396522521972656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,32,8,128,1,float16,float16,0,7.497215906778972
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,32,128,1,float16,float16,0,2.0084053675333657
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,2,128,1,float16,float16,0,1.7708373069763184
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,4,128,1,float16,float16,0,1.8554879824320476
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,4,128,1,float16,float16,0,3.6113065083821616
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,1,128,1,float16,float16,0,1.68669859568278
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,2,128,1,float16,float16,0,3.419818560282389
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,32,128,1,float16,float16,0,1.017514705657959
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,32,8,128,1,float16,float16,0,3.6947625478108725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,1,128,1,float16,float16,0,0.8637440204620361
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,2,128,1,float16,float16,0,0.8876372973124186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,4,128,1,float16,float16,0,0.9446400006612142
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,32,8,128,1,float16,float16,0,1.8503680229187012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,32,128,1,float16,float16,0,0.532309333483378
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,1,128,1,float16,float16,0,0.45738665262858075
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,2,128,1,float16,float16,0,0.46506667137145996
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,4,128,1,float16,float16,0,0.4911786715189616
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,32,8,128,1,float16,float16,0,0.4978346824645996
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,32,8,128,1,float16,float16,0,0.9451519648234049
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,32,128,1,float16,float16,0,0.29047467311223346
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,1,128,1,float16,float16,0,0.2512213389078776
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,2,128,1,float16,float16,0,0.25651200612386066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,4,128,1,float16,float16,0,0.27033599217732746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,32,8,128,1,float16,float16,0,0.2711893320083618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,1,128,1,float16,float16,0,3.8695252736409507
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,2,128,1,float16,float16,0,4.043946584065755
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,32,128,1,float16,float16,0,4.83191458384196
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,32,1,128,1,float16,float16,0,8.07099723815918
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,32,4,128,1,float16,float16,0,8.884735743204752
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,1,128,1,float16,float16,0,1.9785386721293132
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,32,8,128,1,float16,float16,0,9.074687957763672
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,4,128,1,float16,float16,0,2.2422186533610025
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,32,128,1,float16,float16,0,2.407423973083496
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,8,128,1,float16,float16,0,4.401151974995931
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,2,128,1,float16,float16,0,2.0551679929097495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,32,128,1,float16,float16,0,1.2424533367156982
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,1,128,1,float16,float16,0,1.0059093634287517
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,32,8,128,1,float16,float16,0,2.224298636118571
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,4,128,1,float16,float16,0,1.1289599736531575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,2,128,1,float16,float16,0,1.0412373542785645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,32,128,1,float16,float16,0,0.6352213223775228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,1,128,1,float16,float16,0,0.524288018544515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,4,128,1,float16,float16,0,0.5775359869003296
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,32,8,128,1,float16,float16,0,1.13100798924764
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,2,128,1,float16,float16,0,0.5405013163884481
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,32,8,128,1,float16,float16,0,0.5843626658121744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,32,128,1,float16,float16,0,0.33740798632303876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,1,128,1,float16,float16,0,0.28228267033894855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,2,128,1,float16,float16,0,0.289792001247406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,4,128,1,float16,float16,0,0.3097599943478902
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,1,128,1,float16,float16,0,0.1565013329188029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,32,8,128,1,float16,float16,0,0.3107840021451314
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,2,128,1,float16,float16,0,0.1629866659641266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,32,128,1,float16,float16,0,0.187391996383667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,4,128,1,float16,float16,0,0.17407999436060587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,32,8,128,1,float16,float16,0,0.17561600605646768
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,32,4,128,1,float16,float16,0,4.360191980997722
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,32,2,128,1,float16,float16,0,8.407552083333334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,1,128,1,float16,float16,0,3.8777173360188804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,32,128,1,float16,float16,0,5.016405423482259
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,2,128,1,float16,float16,0,4.050431887308757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,4,128,1,float16,float16,0,4.51857058207194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,32,128,1,float16,float16,0,2.5487359364827475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,32,1,128,1,float16,float16,0,7.628288269042969
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,32,8,128,1,float16,float16,0,9.179989496866861
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,32,8,128,1,float16,float16,0,4.532224019368489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,1,128,1,float16,float16,0,1.9498666127522786
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,2,128,1,float16,float16,0,2.096127986907959
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,32,128,1,float16,float16,0,1.3076480229695637
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,1,128,1,float16,float16,0,0.9816746711730957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,8,128,1,float16,float16,0,2.2831786473592124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,32,4,128,1,float16,float16,0,2.2408533096313477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,2,128,1,float16,float16,0,1.0369706948598225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,32,4,128,1,float16,float16,0,9.166165033976236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,4,128,1,float16,float16,0,1.1649706363677979
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,32,128,1,float16,float16,0,0.6533120075861613
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,32,2,128,1,float16,float16,0,8.47701327006022
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,1,128,1,float16,float16,0,0.49937065442403156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,32,8,128,1,float16,float16,0,1.1470506985982258
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,2,128,1,float16,float16,0,0.5278720060984293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,4,128,1,float16,float16,0,0.5790719985961914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,32,128,1,float16,float16,0,0.3418453137079875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,4,128,1,float16,float16,0,0.30344533920288086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,1,128,1,float16,float16,0,0.2653866608937581
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,8,128,1,float16,float16,0,0.311296006043752
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,2,128,1,float16,float16,0,0.15240533153216043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,32,128,1,float16,float16,0,0.18517333269119263
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,4,128,1,float16,float16,0,0.16639999548594156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,8,128,1,float16,float16,0,0.16861865917841592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,32,1,128,1,float16,float16,0,0.1513813336690267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,1,128,1,float16,float16,0,0.0846506655216217
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,32,128,1,float16,float16,0,0.1109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,4,128,1,float16,float16,0,0.09130666653315227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,2,128,1,float16,float16,0,0.08772266904513042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,32,8,128,1,float16,float16,0,0.09523199995358785
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,32,2,128,1,float16,float16,0,0.2810879945755005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,32,8,128,1,float16,float16,0,0.5874346494674683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,1,128,1,float16,float16,0,2.458965301513672
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,32,128,1,float16,float16,0,3.6899840037027993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,2,128,1,float16,float16,0,2.6460159619649253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,4,128,1,float16,float16,0,3.0760958989461265
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,32,1,128,1,float16,float16,0,5.073237419128418
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,32,2,128,1,float16,float16,0,5.367125193277995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,32,128,1,float16,float16,0,1.8479785919189453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,32,128,1,float16,float16,0,0.92740265528361
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,1,128,1,float16,float16,0,1.2392106850941975
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,32,8,128,1,float16,float16,0,3.1073280970255532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,8,128,1,float16,float16,0,1.6047786076863606
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,2,128,1,float16,float16,0,1.3540693918863933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,32,4,128,1,float16,float16,0,1.5709865887959797
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,1,128,1,float16,float16,0,0.6249813238779703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,2,128,1,float16,float16,0,0.6785706679026285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,2,128,1,float16,float16,0,0.34781865278879803
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,32,128,1,float16,float16,0,0.4715520143508911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,8,128,1,float16,float16,0,0.8000853061676025
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,4,128,1,float16,float16,0,0.40174933274586994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,32,4,128,1,float16,float16,0,6.168575922648112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,32,128,1,float16,float16,0,0.24644267559051514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,1,128,1,float16,float16,0,0.3252906600634257
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,32,8,128,1,float16,float16,0,0.40857601165771484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,32,128,1,float16,float16,0,0.13619200388590494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,2,128,1,float16,float16,0,0.1853440006573995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,8,128,1,float16,float16,0,0.21401600042978922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,1,128,1,float16,float16,0,0.17356799046198526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,2,128,1,float16,float16,0,0.10308266679445903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,1,128,1,float16,float16,0,0.09779199957847595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,4,128,1,float16,float16,0,0.11690666278203328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,32,4,128,1,float16,float16,0,0.7818240324656168
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,32,8,128,1,float16,float16,0,0.1181013286113739
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,32,128,1,float16,float16,0,0.07850666840871175
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,2,128,1,float16,float16,0,0.0580266664425532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,1,128,1,float16,float16,0,0.0554666668176651
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,4,128,1,float16,float16,0,0.06229333579540253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,32,128,1,float16,float16,0,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,4,128,1,float16,float16,0,0.040448000033696495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,1,128,1,float16,float16,0,0.037717332442601524
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,2,128,1,float16,float16,0,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,32,8,128,1,float16,float16,0,0.06587733328342438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,32,8,128,1,float16,float16,0,0.04095999896526337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,32,4,128,1,float16,float16,0,0.21333332856496176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,32,128,1,float16,float16,0,1.5156906445821126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,1,128,1,float16,float16,0,0.8913919925689697
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,2,128,1,float16,float16,0,0.9915733337402344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,32,8,128,1,float16,float16,0,6.27507209777832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,32,2,128,1,float16,float16,0,1.969322681427002
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,32,128,1,float16,float16,0,0.762880007425944
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,32,1,128,1,float16,float16,0,1.793706734975179
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,4,128,1,float16,float16,0,1.19705597559611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,2,128,1,float16,float16,0,0.4930560191472371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,32,4,128,1,float16,float16,0,2.404010613759359
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,4,128,1,float16,float16,0,0.5995519955952963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,8,128,1,float16,float16,0,0.6116693417231241
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,1,128,1,float16,float16,0,0.22749867041905722
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,32,8,128,1,float16,float16,0,1.2212906678517659
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,32,128,1,float16,float16,0,0.2001919945081075
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,32,128,1,float16,float16,0,0.39236267407735187
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,4,128,1,float16,float16,0,0.3097599943478902
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,8,128,1,float16,float16,0,0.3165866732597351
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,32,8,128,1,float16,float16,0,2.43285338083903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,2,128,1,float16,float16,0,0.13619200388590494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,8,128,1,float16,float16,0,0.16588800152142844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,4,128,1,float16,float16,0,0.16110933820406595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,2,128,1,float16,float16,0,0.07560533285140991
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,8,128,1,float16,float16,0,0.09096533060073853
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,4,128,1,float16,float16,0,0.08789333701133728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,32,128,1,float16,float16,0,0.10922666390736897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,32,2,128,1,float16,float16,0,0.25497599442799884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,32,128,1,float16,float16,0,0.0631466656923294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,2,128,1,float16,float16,0,0.04215466479460398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,32,1,128,1,float16,float16,0,0.43878400325775146
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,1,128,1,float16,float16,0,0.04027733455101649
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,4,128,1,float16,float16,0,0.04744533201058706
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,32,8,128,1,float16,float16,0,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,32,1,128,1,float16,float16,0,0.12339199582735698
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,32,128,1,float16,float16,0,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,2,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,1,128,1,float16,float16,0,0.02611200014750163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,4,128,1,float16,float16,0,0.0290133332212766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,32,8,128,1,float16,float16,0,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,32,128,1,float16,float16,0,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,4,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,1,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,2,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,32,8,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,32,1,128,1,float16,float16,0,0.07014399766921997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,2,128,1,float16,float16,0,0.39714133739471436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,1,128,1,float16,float16,0,0.3418453137079875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,4,128,1,float16,float16,0,0.5056853294372559
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,8,128,1,float16,float16,0,0.5227520068486532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,32,4,128,1,float16,float16,0,1.1291306813557942
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,1,128,1,float16,float16,0,0.17749333381652832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,32,128,1,float16,float16,0,0.37085866928100586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,2,128,1,float16,float16,0,0.20241065820058188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,32,32,128,1,float16,float16,0,0.7135573228200277
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,8,128,1,float16,float16,0,0.26521599292755127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,32,1,128,1,float16,float16,0,0.7881386280059814
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,32,128,1,float16,float16,0,0.19182932376861572
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,32,8,128,1,float16,float16,0,1.1328852971394856
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,32,4,128,1,float16,float16,0,0.2585600018501282
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,32,2,128,1,float16,float16,0,0.8913919925689697
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,2,128,1,float16,float16,0,0.10871466994285583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,4,128,1,float16,float16,0,0.07406933108965556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,4,128,1,float16,float16,0,0.13448533415794373
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,1,128,1,float16,float16,0,0.056320001681645714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,8,128,1,float16,float16,0,0.0773119976123174
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,1,128,1,float16,float16,0,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,32,128,1,float16,float16,0,0.0580266664425532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,2,128,1,float16,float16,0,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,4,128,1,float16,float16,0,0.037205333511034645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,32,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,32,8,128,1,float16,float16,0,0.04027733455101649
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,1,128,1,float16,float16,0,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,2,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,4,128,1,float16,float16,0,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,32,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,8,128,1,float16,float16,0,0.1384106675783793
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,32,8,128,1,float16,float16,0,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,1,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,32,1,128,1,float16,float16,0,0.0962559978167216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,2,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,8,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,2,128,1,float16,float16,0,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,32,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,32,4,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,4,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,32,32,128,1,float16,float16,0,0.09983999530474345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,8,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,32,2,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,1,128,1,float16,float16,0,0.17442133029301962
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,2,128,1,float16,float16,0,0.20002132654190063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,32,128,1,float16,float16,0,0.3580586512883504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,32,2,128,1,float16,float16,0,0.38570666313171387
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,32,128,1,float16,float16,0,0.18756266434987387
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,8,128,1,float16,float16,0,0.2609493335088094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,32,4,128,1,float16,float16,0,0.49186134338378906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,32,1,128,1,float16,float16,0,0.3327999909718831
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,32,8,128,1,float16,float16,0,0.5070506731669108
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,32,4,128,1,float16,float16,0,0.25514666239420575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,1,128,1,float16,float16,0,0.09710933764775594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,8,128,1,float16,float16,0,0.13738666971524557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,4,128,1,float16,float16,0,0.13499733805656433
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,8,128,1,float16,float16,0,0.07765333354473114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,32,2,128,1,float16,float16,0,0.11025066177050273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,4,128,1,float16,float16,0,0.07423999905586243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,2,128,1,float16,float16,0,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,32,128,1,float16,float16,0,0.09966933727264404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,32,128,1,float16,float16,0,0.056320001681645714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,1,128,1,float16,float16,0,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,4,128,1,float16,float16,0,0.03583999971548716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,32,128,1,float16,float16,0,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,8,128,1,float16,float16,0,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,4,128,1,float16,float16,0,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,1,128,1,float16,float16,0,0.05614933371543884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,32,8,128,1,float16,float16,0,0.03822933385769526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,2,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,32,1,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,32,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,1,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,2,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,32,2,128,1,float16,float16,0,0.06144000093142191
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,8,128,1,float16,float16,0,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,32,4,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,32,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,4,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,8,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,32,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,32,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,32,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,1,128,1,float16,float16,0,0.1599146624406179
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,2,128,1,float16,float16,0,0.17203199863433838
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,4,128,1,float16,float16,0,0.19592533508936563
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,32,128,1,float16,float16,0,0.2315946618715922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,32,1,128,1,float16,float16,0,0.30873600641886395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,32,128,1,float16,float16,0,0.12185600399971008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,1,128,1,float16,float16,0,0.08703999718030293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,32,8,128,1,float16,float16,0,0.19848533471425375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,32,8,128,1,float16,float16,0,0.3845119873682658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,32,2,128,1,float16,float16,0,0.33023999134699505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,32,4,128,1,float16,float16,0,0.37956265608469647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,32,128,1,float16,float16,0,0.06877866884072621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,8,128,1,float16,float16,0,0.10700800021489461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,4,128,1,float16,float16,0,0.10513066252072652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,1,128,1,float16,float16,0,0.04369066655635834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,32,128,1,float16,float16,0,0.03498666733503342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,4,128,1,float16,float16,0,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,4,128,1,float16,float16,0,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,1,128,1,float16,float16,0,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,32,2,128,1,float16,float16,0,0.09233066439628601
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,2,128,1,float16,float16,0,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,32,128,1,float16,float16,0,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,32,8,128,1,float16,float16,0,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,2,128,1,float16,float16,0,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,1,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,2,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,4,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,32,8,128,1,float16,float16,0,0.05444266895453135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,2,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,4,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,32,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,8,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,32,8,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,32,1,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,32,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,32,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,32,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,32,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,1,128,1,float16,float16,0,0.14967466394106546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,2,128,1,float16,float16,0,0.15598932902018228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,32,128,1,float16,float16,0,0.1868799924850464
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,32,1,128,1,float16,float16,0,0.2916693290074666
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,32,2,128,1,float16,float16,0,0.3037866751352946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,32,4,128,1,float16,float16,0,0.32819199562072754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,4,128,1,float16,float16,0,0.16827734311421713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,32,8,128,1,float16,float16,0,0.3322880069414775
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,32,8,128,1,float16,float16,0,0.1704960068066915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,32,128,1,float16,float16,0,0.10035199920336406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,32,128,1,float16,float16,0,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,1,128,1,float16,float16,0,0.07543466488520305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,2,128,1,float16,float16,0,0.04266666869322459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,2,128,1,float16,float16,0,0.07714133461316426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,8,128,1,float16,float16,0,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,4,128,1,float16,float16,0,0.08123733103275299
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,32,8,128,1,float16,float16,0,0.08396800359090169
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,1,128,1,float16,float16,0,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,4,128,1,float16,float16,0,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,32,1,128,1,float16,float16,0,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,32,128,1,float16,float16,0,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,2,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,1,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,4,128,1,float16,float16,0,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,32,8,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,4,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,8,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,32,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,1,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,32,32,128,1,float16,float16,0,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,2,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,4,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,32,8,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,32,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,32,32,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,32,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,0,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,0,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,0,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,32,128,1,float16,float16,0,0.13090133666992188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,32,128,1,float16,float16,0,0.06963199873765309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,0,0.05853866537412008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,0,0.04095999896526337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,0,0.0890880028406779
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,0,0.05205333232879639
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,0,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,32,128,1,float16,float16,0,0.040448000033696495
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,32,128,1,float16,float16,0,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,0,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,32,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,32,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,32,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,32,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,2,128,1,float16,float16,0,36.323158264160156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,1,128,1,float16,float16,0,36.42999521891276
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,24,128,1,float16,float16,0,36.718421936035156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,4,128,1,float16,float16,0,36.639060974121094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,24,128,1,float16,float16,0,18.91362126668294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,1,128,1,float16,float16,0,17.931605021158855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,24,1,128,1,float16,float16,0,72.51558430989583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,24,4,128,1,float16,float16,0,72.61422729492188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,24,8,128,1,float16,float16,0,36.90444691975912
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,24,2,128,1,float16,float16,0,73.47575378417969
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,24,8,128,1,float16,float16,0,73.90071614583333
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,2,128,1,float16,float16,0,18.187605539957683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,4,128,1,float16,float16,0,18.6791254679362
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,24,8,128,1,float16,float16,0,18.277034759521484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,1,128,1,float16,float16,0,9.069226582845053
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,24,128,1,float16,float16,0,9.26907730102539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,2,128,1,float16,float16,0,9.118549346923828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,8,128,1,float16,float16,0,9.447423934936523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,24,4,128,1,float16,float16,0,9.426432291666666
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,24,128,1,float16,float16,0,21.250731150309246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,1,128,1,float16,float16,0,20.523178100585938
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,2,128,1,float16,float16,0,20.797781626383465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,4,128,1,float16,float16,0,21.2138671875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,24,128,1,float16,float16,0,10.640554428100586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,1,128,1,float16,float16,0,10.438485463460287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,2,128,1,float16,float16,0,10.52398935953776
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,24,1,128,1,float16,float16,0,41.100799560546875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,24,2,128,1,float16,float16,0,41.96300760904948
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,24,4,128,1,float16,float16,0,41.59556325276693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,24,8,128,1,float16,float16,0,41.51483662923177
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,24,128,1,float16,float16,0,5.256362597147624
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,2,128,1,float16,float16,0,5.144576072692871
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,4,128,1,float16,float16,0,5.1514027913411455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,1,128,1,float16,float16,0,5.062656084696452
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,24,8,128,1,float16,float16,0,21.042517344156902
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,4,128,1,float16,float16,0,10.30126953125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,24,8,128,1,float16,float16,0,10.541056315104166
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,24,8,128,1,float16,float16,0,5.275477409362793
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,24,128,1,float16,float16,0,15.05740737915039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,1,128,1,float16,float16,0,14.363136291503906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,2,128,1,float16,float16,0,14.634325663248697
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,4,128,1,float16,float16,0,14.610602060953775
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,24,128,1,float16,float16,0,7.481173197428386
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,1,128,1,float16,float16,0,7.133525212605794
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,2,128,1,float16,float16,0,7.170218785603841
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,24,1,128,1,float16,float16,0,29.189119974772137
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,24,4,128,1,float16,float16,0,29.16454315185547
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,24,2,128,1,float16,float16,0,29.212671915690105
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,24,8,128,1,float16,float16,0,29.40979258219401
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,24,8,128,1,float16,float16,0,14.85687510172526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,24,128,1,float16,float16,0,3.8027947743733725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,1,128,1,float16,float16,0,3.7867520650227866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,2,128,1,float16,float16,0,3.616938591003418
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,4,128,1,float16,float16,0,3.8062079747517905
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,4,128,1,float16,float16,0,7.35812250773112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,24,8,128,1,float16,float16,0,7.455061594645183
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,24,8,128,1,float16,float16,0,3.6778666178385415
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,1,128,1,float16,float16,0,18.538837432861328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,2,128,1,float16,float16,0,18.87488047281901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,24,128,1,float16,float16,0,19.677525838216145
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,4,128,1,float16,float16,0,19.04418182373047
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,1,128,1,float16,float16,0,9.300650914510092
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,24,128,1,float16,float16,0,9.73687489827474
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,2,128,1,float16,float16,0,9.351168314615885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,24,1,128,1,float16,float16,0,37.18775431315104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,24,2,128,1,float16,float16,0,36.93584950764974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,24,128,1,float16,float16,0,4.8395945231119795
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,24,8,128,1,float16,float16,0,37.84567515055338
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,24,4,128,1,float16,float16,0,38.83656565348307
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,24,8,128,1,float16,float16,0,19.28652826944987
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,1,128,1,float16,float16,0,4.550997416178386
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,2,128,1,float16,float16,0,4.612949371337891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,4,128,1,float16,float16,0,9.611775716145834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,24,8,128,1,float16,float16,0,9.682261149088541
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,24,128,1,float16,float16,0,2.5224533081054688
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,1,128,1,float16,float16,0,2.385237375895182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,4,128,1,float16,float16,0,4.71500809987386
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,24,8,128,1,float16,float16,0,4.745898564656575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,2,128,1,float16,float16,0,2.3891626993815103
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,4,128,1,float16,float16,0,2.4427520434061685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,24,8,128,1,float16,float16,0,2.4685227076212564
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,1,128,1,float16,float16,0,10.683391571044922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,2,128,1,float16,float16,0,10.597546895345053
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,4,128,1,float16,float16,0,11.068416595458984
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,24,128,1,float16,float16,0,11.142144521077475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,24,128,1,float16,float16,0,5.598208109537761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,1,128,1,float16,float16,0,5.248000144958496
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,2,128,1,float16,float16,0,5.188607851664226
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,24,1,128,1,float16,float16,0,21.478228251139324
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,24,2,128,1,float16,float16,0,21.629781087239582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,24,4,128,1,float16,float16,0,22.052523295084637
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,24,8,128,1,float16,float16,0,21.9497807820638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,24,8,128,1,float16,float16,0,10.99178695678711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,24,128,1,float16,float16,0,2.859861373901367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,1,128,1,float16,float16,0,2.677248001098633
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,2,128,1,float16,float16,0,2.671104113260905
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,4,128,1,float16,float16,0,5.449898401896159
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,24,8,128,1,float16,float16,0,5.504341125488281
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,4,128,1,float16,float16,0,2.7648000717163086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,24,128,1,float16,float16,0,1.4718292554219563
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,1,128,1,float16,float16,0,1.4144852956136067
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,2,128,1,float16,float16,0,1.3805227279663086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,4,128,1,float16,float16,0,1.4337706565856934
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,24,8,128,1,float16,float16,0,2.76906681060791
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,24,8,128,1,float16,float16,0,1.4621013005574544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,1,128,1,float16,float16,0,9.728853225708008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,2,128,1,float16,float16,0,10.062677383422852
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,24,128,1,float16,float16,0,10.52945073445638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,24,128,1,float16,float16,0,5.1858774820963545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,1,128,1,float16,float16,0,4.620800018310547
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,24,2,128,1,float16,float16,0,19.887274424235027
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,24,4,128,1,float16,float16,0,20.707327524820965
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,4,128,1,float16,float16,0,10.287786483764648
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,24,8,128,1,float16,float16,0,10.51477305094401
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,2,128,1,float16,float16,0,4.742485364278157
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,24,128,1,float16,float16,0,2.6603520711263022
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,1,128,1,float16,float16,0,2.415445327758789
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,4,128,1,float16,float16,0,4.963328043619792
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,2,128,1,float16,float16,0,2.483882745107015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,24,1,128,1,float16,float16,0,19.464874267578125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,24,8,128,1,float16,float16,0,5.141162554423015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,24,8,128,1,float16,float16,0,20.667221069335938
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,4,128,1,float16,float16,0,2.545152028401693
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,24,128,1,float16,float16,0,1.366528034210205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,24,8,128,1,float16,float16,0,2.52620792388916
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,2,128,1,float16,float16,0,1.2789759635925293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,1,128,1,float16,float16,0,1.2136106491088867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,4,128,1,float16,float16,0,1.3660160700480144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,24,128,1,float16,float16,0,0.7475199699401855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,24,8,128,1,float16,float16,0,1.3134506543477376
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,2,128,1,float16,float16,0,0.6683306694030762
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,1,128,1,float16,float16,0,0.6625279982884725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,4,128,1,float16,float16,0,0.6901760101318359
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,24,8,128,1,float16,float16,0,0.6999039649963379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,2,128,1,float16,float16,0,5.758975982666016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,4,128,1,float16,float16,0,5.978623708089192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,24,128,1,float16,float16,0,6.2300160725911455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,1,128,1,float16,float16,0,5.4570668538411455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,1,128,1,float16,float16,0,2.694485346476237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,24,128,1,float16,float16,0,3.090261459350586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,2,128,1,float16,float16,0,2.8093439737955728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,24,2,128,1,float16,float16,0,11.585365295410156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,24,4,128,1,float16,float16,0,12.082346598307291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,24,1,128,1,float16,float16,0,11.142655690511068
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,24,8,128,1,float16,float16,0,12.15453847249349
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,24,8,128,1,float16,float16,0,6.064640045166016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,1,128,1,float16,float16,0,1.4522026379903157
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,24,128,1,float16,float16,0,1.550165335337321
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,2,128,1,float16,float16,0,1.4504960378011067
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,4,128,1,float16,float16,0,2.9385385513305664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,24,128,1,float16,float16,0,0.8536746501922607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,1,128,1,float16,float16,0,0.731818675994873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,8,128,1,float16,float16,0,1.5240532557169597
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,24,8,128,1,float16,float16,0,2.9837652842203775
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,24,4,128,1,float16,float16,0,1.5042559305826824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,4,128,1,float16,float16,0,0.7785813013712565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,2,128,1,float16,float16,0,0.7406933307647705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,24,128,1,float16,float16,0,0.44697598616282147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,24,8,128,1,float16,float16,0,0.7922346591949463
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,1,128,1,float16,float16,0,0.3993599812189738
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,2,128,1,float16,float16,0,0.4072106679280599
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,4,128,1,float16,float16,0,0.42769066492716473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,24,8,128,1,float16,float16,0,0.423253337542216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,24,128,1,float16,float16,0,6.088703791300456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,4,128,1,float16,float16,0,5.64411735534668
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,1,128,1,float16,float16,0,5.1548159917195635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,2,128,1,float16,float16,0,5.311658541361491
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,1,128,1,float16,float16,0,2.527402718861898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,24,128,1,float16,float16,0,2.9905920028686523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,24,1,128,1,float16,float16,0,10.660693486531576
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,24,4,128,1,float16,float16,0,11.9094607035319
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,24,8,128,1,float16,float16,0,11.841023763020834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,2,128,1,float16,float16,0,2.664618651072184
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,1,128,1,float16,float16,0,1.3110613028208415
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,24,8,128,1,float16,float16,0,5.716821034749349
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,4,128,1,float16,float16,0,2.8339198430379233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,2,128,1,float16,float16,0,1.3470719655354817
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,24,128,1,float16,float16,0,1.546240011850993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,24,8,128,1,float16,float16,0,2.8269227345784507
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,4,128,1,float16,float16,0,1.449130694071452
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,1,128,1,float16,float16,0,0.6608213186264038
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,24,2,128,1,float16,float16,0,10.697898864746094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,24,128,1,float16,float16,0,0.7975253264109293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,24,8,128,1,float16,float16,0,1.458346684773763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,4,128,1,float16,float16,0,0.7406933307647705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,24,128,1,float16,float16,0,0.41625599066416424
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,1,128,1,float16,float16,0,0.3563520113627116
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,2,128,1,float16,float16,0,0.6881279945373535
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,24,8,128,1,float16,float16,0,0.7444480260213217
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,24,128,1,float16,float16,0,0.23040000597635904
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,2,128,1,float16,float16,0,0.20565332969029745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,8,128,1,float16,float16,0,0.3968000014623006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,4,128,1,float16,float16,0,0.3964586655298869
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,4,128,1,float16,float16,0,0.2206719915072123
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,1,128,1,float16,float16,0,0.20309333006540933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,24,8,128,1,float16,float16,0,0.22374399503072104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,24,2,128,1,float16,float16,0,0.36932265758514404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,1,128,1,float16,float16,0,2.9777921040852866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,24,128,1,float16,float16,0,3.7073920567830405
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,2,128,1,float16,float16,0,3.1332693099975586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,24,1,128,1,float16,float16,0,5.903701146443685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,24,4,128,1,float16,float16,0,6.999722798665364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,24,8,128,1,float16,float16,0,6.964053471883138
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,24,2,128,1,float16,float16,0,6.134613037109375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,4,128,1,float16,float16,0,1.7257812817891438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,8,128,1,float16,float16,0,3.4802347819010415
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,1,128,1,float16,float16,0,1.4899199803670247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,24,4,128,1,float16,float16,0,3.3962666193644204
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,24,128,1,float16,float16,0,1.8599252700805664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,24,128,1,float16,float16,0,0.951637347539266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,2,128,1,float16,float16,0,1.5796906153361003
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,24,8,128,1,float16,float16,0,1.7421654065450032
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,2,128,1,float16,float16,0,0.8026453653971354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,1,128,1,float16,float16,0,0.7661226590474447
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,4,128,1,float16,float16,0,0.8913919925689697
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,24,128,1,float16,float16,0,0.49544533093770343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,1,128,1,float16,float16,0,0.40447998046875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,2,128,1,float16,float16,0,0.4203519821166992
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,24,128,1,float16,float16,0,0.2723840077718099
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,24,8,128,1,float16,float16,0,0.897706667582194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,4,128,1,float16,float16,0,0.45789865652720135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,24,8,128,1,float16,float16,0,0.46557867527008057
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,2,128,1,float16,float16,0,0.22988800207773843
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,1,128,1,float16,float16,0,0.22050132354100546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,1,128,1,float16,float16,0,0.12526933352152506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,4,128,1,float16,float16,0,0.24968532721201578
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,24,8,128,1,float16,float16,0,0.25497599442799884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,2,128,1,float16,float16,0,0.13141333063443503
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,24,128,1,float16,float16,0,0.1518933375676473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,8,128,1,float16,float16,0,0.14335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,24,4,128,1,float16,float16,0,0.13875200351079306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,24,128,1,float16,float16,0,3.853311856587728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,1,128,1,float16,float16,0,2.893141428629557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,24,8,128,1,float16,float16,0,7.1538346608479815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,2,128,1,float16,float16,0,3.100501378377279
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,1,128,1,float16,float16,0,1.4701226552327473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,2,128,1,float16,float16,0,1.5697919527689617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,24,128,1,float16,float16,0,1.9541333516438801
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,24,2,128,1,float16,float16,0,6.32473627726237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,8,128,1,float16,float16,0,3.596288045247396
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,24,4,128,1,float16,float16,0,3.507370630900065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,24,1,128,1,float16,float16,0,5.87229855855306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,24,128,1,float16,float16,0,0.9929386774698893
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,24,4,128,1,float16,float16,0,7.198037465413411
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,8,128,1,float16,float16,0,1.798997402191162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,2,128,1,float16,float16,0,0.8026453653971354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,24,128,1,float16,float16,0,0.5116586685180664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,24,4,128,1,float16,float16,0,1.7812479337056477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,2,128,1,float16,float16,0,0.4113066593805949
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,8,128,1,float16,float16,0,0.9217706521352133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,4,128,1,float16,float16,0,0.46609067916870117
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,8,128,1,float16,float16,0,0.47138134638468426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,4,128,1,float16,float16,0,0.8965120315551758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,24,1,128,1,float16,float16,0,0.7424000104268392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,1,128,1,float16,float16,0,0.20804266134897867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,8,128,1,float16,float16,0,0.2505386670430501
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,4,128,1,float16,float16,0,0.24576000372568765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,2,128,1,float16,float16,0,0.11963733037312825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,24,128,1,float16,float16,0,0.2696533401807149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,24,2,128,1,float16,float16,0,0.2218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,1,128,1,float16,float16,0,0.11929600437482198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,24,128,1,float16,float16,0,0.1508693297704061
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,4,128,1,float16,float16,0,0.13909332950909933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,1,128,1,float16,float16,0,0.0718506673971812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,24,128,1,float16,float16,0,0.08994133273760478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,24,8,128,1,float16,float16,0,0.14165332913398743
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,4,128,1,float16,float16,0,0.07816533247629802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,2,128,1,float16,float16,0,0.07543466488520305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,24,8,128,1,float16,float16,0,0.08038400113582611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,24,1,128,1,float16,float16,0,0.3872426748275757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,1,128,1,float16,float16,0,1.8788693745930989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,24,128,1,float16,float16,0,2.8282880783081055
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,2,128,1,float16,float16,0,2.066431999206543
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,4,128,1,float16,float16,0,2.4959999720255532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,24,128,1,float16,float16,0,1.4385493596394856
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,1,128,1,float16,float16,0,0.9442986647288004
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,24,1,128,1,float16,float16,0,3.763200124104818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,24,2,128,1,float16,float16,0,4.108458518981934
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,2,128,1,float16,float16,0,1.0513066450754802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,24,8,128,1,float16,float16,0,2.5371306737264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,24,128,1,float16,float16,0,0.7391573588053385
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,1,128,1,float16,float16,0,0.4891306559244792
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,4,128,1,float16,float16,0,1.2661759853363037
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,24,4,128,1,float16,float16,0,4.976810773213704
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,24,8,128,1,float16,float16,0,1.2779520352681477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,2,128,1,float16,float16,0,0.5316266616185507
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,24,128,1,float16,float16,0,0.37597866853078205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,24,128,1,float16,float16,0,0.20070399840672812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,8,128,1,float16,float16,0,0.6572373310724894
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,8,128,1,float16,float16,0,0.3346773386001587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,24,8,128,1,float16,float16,0,5.100202560424805
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,1,128,1,float16,float16,0,0.2527573307355245
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,2,128,1,float16,float16,0,0.14984533190727234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,4,128,1,float16,float16,0,0.33023999134699505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,1,128,1,float16,float16,0,0.13738666971524557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,24,128,1,float16,float16,0,0.10956799983978271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,8,128,1,float16,float16,0,0.17937066157658896
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,2,128,1,float16,float16,0,0.08157866696516673
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,1,128,1,float16,float16,0,0.07577600081761678
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,4,128,1,float16,float16,0,0.09693866968154907
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,24,4,128,1,float16,float16,0,0.6367573340733846
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,24,8,128,1,float16,float16,0,0.09949866930643718
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,2,128,1,float16,float16,0,0.04915200173854828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,1,128,1,float16,float16,0,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,24,128,1,float16,float16,0,0.06621866424878438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,24,2,128,1,float16,float16,0,0.27801599105199176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,8,128,1,float16,float16,0,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,24,4,128,1,float16,float16,0,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,24,128,1,float16,float16,0,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,1,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,2,128,1,float16,float16,0,0.0290133332212766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,8,128,1,float16,float16,0,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,24,4,128,1,float16,float16,0,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,24,4,128,1,float16,float16,0,0.17390932639439902
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,1,128,1,float16,float16,0,0.6882987022399902
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,24,128,1,float16,float16,0,1.1784533659617107
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,24,1,128,1,float16,float16,0,1.3743786811828613
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,24,2,128,1,float16,float16,0,1.567914644877116
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,24,4,128,1,float16,float16,0,1.9886080423990886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,2,128,1,float16,float16,0,0.7884799639383951
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,1,128,1,float16,float16,0,0.34013867378234863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,4,128,1,float16,float16,0,0.99618132909139
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,2,128,1,float16,float16,0,0.3947519858678182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,24,8,128,1,float16,float16,0,1.0180266698201497
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,4,128,1,float16,float16,0,0.49800535043080646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,24,128,1,float16,float16,0,0.3058346708615621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,24,128,1,float16,float16,0,0.5949440002441406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,24,8,128,1,float16,float16,0,0.5102933247884115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,2,128,1,float16,float16,0,0.2053119937578837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,1,128,1,float16,float16,0,0.17971199750900269
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,24,128,1,float16,float16,0,0.16247466206550598
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,8,128,1,float16,float16,0,0.2657279968261719
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,4,128,1,float16,float16,0,0.13636266191800436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,24,128,1,float16,float16,0,0.08942932883898418
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,8,128,1,float16,float16,0,0.14062933127085367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,1,128,1,float16,float16,0,0.09864532947540283
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,2,128,1,float16,float16,0,0.06007466713587443
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,1,128,1,float16,float16,0,0.05495466788609823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,4,128,1,float16,float16,0,0.07543466488520305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,24,2,128,1,float16,float16,0,0.11076266566912334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,24,8,128,1,float16,float16,0,0.07833600044250488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,24,128,1,float16,float16,0,0.05120000243186951
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,1,128,1,float16,float16,0,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,24,4,128,1,float16,float16,0,0.2590720057487488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,2,128,1,float16,float16,0,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,8,128,1,float16,float16,0,0.03925333420435587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,24,4,128,1,float16,float16,0,0.038058665891488395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,2,128,1,float16,float16,0,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,1,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,24,128,1,float16,float16,0,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,2,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,4,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,8,128,1,float16,float16,0,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,24,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,1,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,24,4,128,1,float16,float16,0,0.025087999800841015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,24,8,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,24,8,128,1,float16,float16,0,2.040320078531901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,2,128,1,float16,float16,0,0.3280213276545207
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,4,128,1,float16,float16,0,0.4322986602783203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,1,128,1,float16,float16,0,0.26606933275858563
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,24,1,128,1,float16,float16,0,0.6166186730066935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,24,4,128,1,float16,float16,0,0.9280853271484375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,1,128,1,float16,float16,0,0.14079999923706055
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,24,128,1,float16,float16,0,0.28757333755493164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,8,128,1,float16,float16,0,0.45073068141937256
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,24,2,128,1,float16,float16,0,0.7080960273742676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,8,128,1,float16,float16,0,0.22971733411153158
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,24,8,128,1,float16,float16,0,0.9635840257008871
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,24,24,128,1,float16,float16,0,0.5705386797587076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,24,128,1,float16,float16,0,0.14916266997655234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,2,128,1,float16,float16,0,0.16657066345214844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,4,128,1,float16,float16,0,0.11622400085131328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,1,128,1,float16,float16,0,0.07901866734027863
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,24,4,128,1,float16,float16,0,0.22169599930445352
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,4,128,1,float16,float16,0,0.06382933259010315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,2,128,1,float16,float16,0,0.05034666756788889
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,8,128,1,float16,float16,0,0.06877866884072621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,1,128,1,float16,float16,0,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,2,128,1,float16,float16,0,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,1,128,1,float16,float16,0,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,4,128,1,float16,float16,0,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,8,128,1,float16,float16,0,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,24,24,128,1,float16,float16,0,0.04454400142033895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,24,128,1,float16,float16,0,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,2,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,4,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,1,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,24,128,1,float16,float16,0,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,1,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,24,8,128,1,float16,float16,0,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,2,128,1,float16,float16,0,0.09113599856694539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,2,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,4,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,24,8,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,24,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,4,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,24,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,24,8,128,1,float16,float16,0,0.12236799796422322
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,24,24,128,1,float16,float16,0,0.08345599969228108
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,1,128,1,float16,float16,0,0.1389226714769999
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,2,128,1,float16,float16,0,0.16657066345214844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,24,128,1,float16,float16,0,0.2868906656901042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,4,128,1,float16,float16,0,0.21794132391611734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,24,4,128,1,float16,float16,0,0.4242773453394572
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,24,2,128,1,float16,float16,0,0.31522132953008014
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,24,8,128,1,float16,float16,0,0.2295466661453247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,24,1,128,1,float16,float16,0,0.2629973292350769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,24,8,128,1,float16,float16,0,0.44356266657511395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,1,128,1,float16,float16,0,0.07867733140786488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,24,128,1,float16,float16,0,0.1474560002485911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,2,128,1,float16,float16,0,0.09079466263453166
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,2,128,1,float16,float16,0,0.048469334840774536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,8,128,1,float16,float16,0,0.06860800087451935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,8,128,1,float16,float16,0,0.12100266416867574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,24,128,1,float16,float16,0,0.08226133386294048
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,1,128,1,float16,float16,0,0.025429333249727886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,24,4,128,1,float16,float16,0,0.1153706709543864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,4,128,1,float16,float16,0,0.06519466638565063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,2,128,1,float16,float16,0,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,24,128,1,float16,float16,0,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,24,128,1,float16,float16,0,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,2,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,4,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,24,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,4,128,1,float16,float16,0,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,24,8,128,1,float16,float16,0,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,8,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,24,1,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,24,1,128,1,float16,float16,0,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,2,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,4,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,24,8,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,24,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,24,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,24,8,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,8,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,24,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,1,128,1,float16,float16,0,0.12680533528327942
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,2,128,1,float16,float16,0,0.1378986636797587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,4,128,1,float16,float16,0,0.16196266810099283
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,24,1,128,1,float16,float16,0,0.2397866646448771
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,24,2,128,1,float16,float16,0,0.2606079975763957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,24,128,1,float16,float16,0,0.18551466862360635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,1,128,1,float16,float16,0,0.06553600231806438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,24,8,128,1,float16,float16,0,0.16537599762280783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,24,4,128,1,float16,float16,0,0.3145386576652527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,24,128,1,float16,float16,0,0.09966933727264404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,24,8,128,1,float16,float16,0,0.318122665087382
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,24,128,1,float16,float16,0,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,2,128,1,float16,float16,0,0.0727040022611618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,24,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,4,128,1,float16,float16,0,0.08772266904513042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,4,128,1,float16,float16,0,0.04266666869322459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,1,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,1,128,1,float16,float16,0,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,2,128,1,float16,float16,0,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,8,128,1,float16,float16,0,0.02628266563018163
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,24,128,1,float16,float16,0,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,2,128,1,float16,float16,0,0.03822933385769526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,24,4,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,24,8,128,1,float16,float16,0,0.09028266867001851
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,1,128,1,float16,float16,0,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,2,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,4,128,1,float16,float16,0,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,24,8,128,1,float16,float16,0,0.04386133452256521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,24,8,128,1,float16,float16,0,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,8,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,24,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,24,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,24,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,24,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,24,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,24,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,1,128,1,float16,float16,0,0.11400533715883891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,2,128,1,float16,float16,0,0.12288000186284383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,24,128,1,float16,float16,0,0.1474560002485911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,24,4,128,1,float16,float16,0,0.26282666126887005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,24,1,128,1,float16,float16,0,0.2259626587231954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,24,8,128,1,float16,float16,0,0.2658986647923787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,4,128,1,float16,float16,0,0.13619200388590494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,24,2,128,1,float16,float16,0,0.23859200874964395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,24,128,1,float16,float16,0,0.0795306662718455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,24,8,128,1,float16,float16,0,0.13926399747530618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,24,128,1,float16,float16,0,0.04027733455101649
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,1,128,1,float16,float16,0,0.059903999169667564
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,8,128,1,float16,float16,0,0.06673066814740498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,2,128,1,float16,float16,0,0.06144000093142191
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,2,128,1,float16,float16,0,0.03601066768169403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,24,4,128,1,float16,float16,0,0.06604800124963124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,1,128,1,float16,float16,0,0.034474665919939675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,24,128,1,float16,float16,0,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,8,128,1,float16,float16,0,0.03822933385769526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,8,128,1,float16,float16,0,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,2,128,1,float16,float16,0,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,1,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,24,4,128,1,float16,float16,0,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,24,4,128,1,float16,float16,0,0.03788800040880839
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,24,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,4,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,8,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,2,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,24,1,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,24,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,24,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,24,24,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,24,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,24,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,24,128,1,float16,float16,0,0.10205866893132527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,0,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,0,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,0,0.05563733478387197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,24,128,1,float16,float16,0,0.05597866574923197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,0,0.0865280032157898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,0,0.03839999934037527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,0,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,0,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,24,128,1,float16,float16,0,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,0,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,0,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,0,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,24,128,1,float16,float16,0,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,24,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,24,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,24,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,24,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,2,128,1,float16,float16,0,23.83667246500651
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,1,128,1,float16,float16,0,24.484522501627605
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,4,128,1,float16,float16,0,24.717140197753906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,16,128,1,float16,float16,0,24.783360799153645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,1,128,1,float16,float16,0,12.215466817220053
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,16,128,1,float16,float16,0,12.684288024902344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,2,128,1,float16,float16,0,12.334932963053385
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,16,128,1,float16,float16,0,6.265514373779297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,16,1,128,1,float16,float16,0,48.5372568766276
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,16,2,128,1,float16,float16,0,48.89873250325521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,16,8,128,1,float16,float16,0,24.68420155843099
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,16,8,128,1,float16,float16,0,49.56023661295573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,16,4,128,1,float16,float16,0,48.9345703125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,4,128,1,float16,float16,0,12.51583989461263
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,16,8,128,1,float16,float16,0,12.655104319254557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,1,128,1,float16,float16,0,6.139733632405599
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,2,128,1,float16,float16,0,6.224213282267253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,4,128,1,float16,float16,0,6.182570775349935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,16,8,128,1,float16,float16,0,6.302207946777344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,16,128,1,float16,float16,0,14.55086898803711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,1,128,1,float16,float16,0,13.555370330810547
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,2,128,1,float16,float16,0,13.92145029703776
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,16,1,128,1,float16,float16,0,27.2005132039388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,16,128,1,float16,float16,0,7.119359970092773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,1,128,1,float16,float16,0,6.93998908996582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,16,2,128,1,float16,float16,0,27.639124552408855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,16,4,128,1,float16,float16,0,28.678143819173176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,16,8,128,1,float16,float16,0,28.38971710205078
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,16,128,1,float16,float16,0,3.660458564758301
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,4,128,1,float16,float16,0,14.169600168863932
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,1,128,1,float16,float16,0,3.5471359888712564
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,2,128,1,float16,float16,0,3.5532798767089844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,16,8,128,1,float16,float16,0,14.489941914876303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,2,128,1,float16,float16,0,7.071061452229817
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,4,128,1,float16,float16,0,7.019349416097005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,16,8,128,1,float16,float16,0,7.171413421630859
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,4,128,1,float16,float16,0,3.573589324951172
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,16,8,128,1,float16,float16,0,3.564373334248861
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,1,128,1,float16,float16,0,9.596245447794596
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,16,128,1,float16,float16,0,10.010112126668295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,2,128,1,float16,float16,0,9.728170394897461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,4,128,1,float16,float16,0,9.810431798299154
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,16,128,1,float16,float16,0,4.953258514404297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,16,1,128,1,float16,float16,0,19.35445276896159
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,1,128,1,float16,float16,0,4.929706573486328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,16,2,128,1,float16,float16,0,19.397973378499348
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,2,128,1,float16,float16,0,4.8192853927612305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,16,4,128,1,float16,float16,0,19.94274139404297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,16,8,128,1,float16,float16,0,19.776512145996094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,16,8,128,1,float16,float16,0,10.008405049641928
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,16,128,1,float16,float16,0,2.605397383371989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,4,128,1,float16,float16,0,4.932949384053548
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,1,128,1,float16,float16,0,2.489173412322998
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,2,128,1,float16,float16,0,2.542762597401937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,4,128,1,float16,float16,0,2.5895253817240396
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,16,8,128,1,float16,float16,0,2.5688746770222983
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,16,8,128,1,float16,float16,0,4.997119903564453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,1,128,1,float16,float16,0,12.564992268880209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,2,128,1,float16,float16,0,12.460714975992838
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,16,128,1,float16,float16,0,13.232298533121744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,4,128,1,float16,float16,0,13.22103500366211
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,1,128,1,float16,float16,0,6.124202728271484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,16,128,1,float16,float16,0,6.456319808959961
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,2,128,1,float16,float16,0,6.34880002339681
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,16,1,128,1,float16,float16,0,24.944297790527344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,16,2,128,1,float16,float16,0,25.07520039876302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,16,4,128,1,float16,float16,0,25.620992024739582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,16,128,1,float16,float16,0,3.4739198684692383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,16,8,128,1,float16,float16,0,26.001920064290363
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,16,8,128,1,float16,float16,0,13.273600260416666
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,1,128,1,float16,float16,0,3.0694398880004883
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,8,128,1,float16,float16,0,6.393856048583984
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,16,4,128,1,float16,float16,0,6.449151992797852
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,2,128,1,float16,float16,0,3.1778132120768228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,16,128,1,float16,float16,0,1.7558186848958333
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,4,128,1,float16,float16,0,3.2322559356689453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,1,128,1,float16,float16,0,1.702229340871175
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,16,8,128,1,float16,float16,0,3.291818618774414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,2,128,1,float16,float16,0,1.6667307217915852
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,4,128,1,float16,float16,0,1.6989866892496746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,16,8,128,1,float16,float16,0,1.712469259897868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,1,128,1,float16,float16,0,7.065600077311198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,2,128,1,float16,float16,0,6.982826868693034
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,16,128,1,float16,float16,0,7.662591934204102
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,4,128,1,float16,float16,0,7.505578358968099
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,16,128,1,float16,float16,0,3.7968212763468423
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,1,128,1,float16,float16,0,3.5196587244669595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,2,128,1,float16,float16,0,3.5554987589518228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,16,1,128,1,float16,float16,0,14.10867182413737
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,16,2,128,1,float16,float16,0,14.423210144042969
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,16,4,128,1,float16,float16,0,15.009962717692057
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,16,8,128,1,float16,float16,0,15.283028920491537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,16,8,128,1,float16,float16,0,7.530495961507161
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,16,128,1,float16,float16,0,1.9450880686442058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,1,128,1,float16,float16,0,1.7921706835428874
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,2,128,1,float16,float16,0,1.8542933464050293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,4,128,1,float16,float16,0,1.923583984375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,16,128,1,float16,float16,0,1.0695679982503254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,1,128,1,float16,float16,0,0.9589760303497314
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,16,8,128,1,float16,float16,0,1.9834879239400227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,4,128,1,float16,float16,0,3.6367359161376953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,2,128,1,float16,float16,0,0.9734826882680258
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,16,8,128,1,float16,float16,0,3.721045176188151
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,4,128,1,float16,float16,0,1.024341344833374
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,16,8,128,1,float16,float16,0,1.016319990158081
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,2,128,1,float16,float16,0,6.648661295572917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,1,128,1,float16,float16,0,6.532096227010091
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,16,128,1,float16,float16,0,7.195648193359375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,4,128,1,float16,float16,0,6.919680277506511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,16,128,1,float16,float16,0,3.6172800064086914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,1,128,1,float16,float16,0,3.1493120193481445
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,2,128,1,float16,float16,0,3.260416030883789
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,16,1,128,1,float16,float16,0,13.173248291015625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,16,2,128,1,float16,float16,0,13.422250111897787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,16,4,128,1,float16,float16,0,14.268756866455078
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,16,8,128,1,float16,float16,0,14.063616434733072
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,16,8,128,1,float16,float16,0,7.05245844523112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,16,128,1,float16,float16,0,1.8553172747294109
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,1,128,1,float16,float16,0,1.639253298441569
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,4,128,1,float16,float16,0,3.47869873046875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,16,8,128,1,float16,float16,0,3.495936075846354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,2,128,1,float16,float16,0,1.7361920674641926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,4,128,1,float16,float16,0,1.791829268137614
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,16,8,128,1,float16,float16,0,1.8030932744344075
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,16,128,1,float16,float16,0,0.9540266990661621
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,1,128,1,float16,float16,0,0.8654507001241049
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,2,128,1,float16,float16,0,0.9006079832712809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,4,128,1,float16,float16,0,0.9316693147023519
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,16,128,1,float16,float16,0,0.5205333232879639
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,16,8,128,1,float16,float16,0,0.9594879945119222
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,1,128,1,float16,float16,0,0.474453330039978
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,2,128,1,float16,float16,0,0.48179201285044354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,8,128,1,float16,float16,0,0.508074680964152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,16,4,128,1,float16,float16,0,0.5072213411331177
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,16,128,1,float16,float16,0,4.240384101867676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,2,128,1,float16,float16,0,3.7954559326171875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,1,128,1,float16,float16,0,3.6183039347330728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,16,2,128,1,float16,float16,0,7.636821111043294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,16,1,128,1,float16,float16,0,7.400789260864258
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,16,128,1,float16,float16,0,2.1352106730143228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,16,4,128,1,float16,float16,0,8.370517094930014
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,16,8,128,1,float16,float16,0,8.436053593953451
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,4,128,1,float16,float16,0,4.060842514038086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,16,8,128,1,float16,float16,0,4.133887926737468
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,16,128,1,float16,float16,0,1.1168426672617595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,1,128,1,float16,float16,0,1.8549760182698567
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,2,128,1,float16,float16,0,0.9995946884155273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,8,128,1,float16,float16,0,2.1039786338806152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,4,128,1,float16,float16,0,2.074282646179199
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,16,2,128,1,float16,float16,0,1.9174399375915527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,4,128,1,float16,float16,0,1.0763946374257405
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,1,128,1,float16,float16,0,0.9337173302968343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,1,128,1,float16,float16,0,0.5142186482747396
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,16,128,1,float16,float16,0,0.5858986775080363
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,16,8,128,1,float16,float16,0,1.0833919843037922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,2,128,1,float16,float16,0,0.5348693529764811
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,4,128,1,float16,float16,0,0.570026675860087
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,16,8,128,1,float16,float16,0,0.5713920195897421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,16,128,1,float16,float16,0,0.3251199920972188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,2,128,1,float16,float16,0,0.2940586606661479
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,4,128,1,float16,float16,0,0.318122665087382
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,8,128,1,float16,float16,0,0.3206826647122701
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,16,1,128,1,float16,float16,0,0.28484266996383667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,1,128,1,float16,float16,0,3.3989973068237305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,16,128,1,float16,float16,0,4.21563720703125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,2,128,1,float16,float16,0,3.597482681274414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,4,128,1,float16,float16,0,3.9772160847981772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,16,128,1,float16,float16,0,2.1288960774739585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,16,1,128,1,float16,float16,0,7.162197113037109
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,1,128,1,float16,float16,0,1.71997865041097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,16,4,128,1,float16,float16,0,8.102229436238607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,16,8,128,1,float16,float16,0,8.214357376098633
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,16,128,1,float16,float16,0,1.0825386842091878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,16,8,128,1,float16,float16,0,4.036095937093099
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,2,128,1,float16,float16,0,1.8206720352172852
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,1,128,1,float16,float16,0,0.8835413455963135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,4,128,1,float16,float16,0,2.0147199630737305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,16,8,128,1,float16,float16,0,2.0831573804219565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,4,128,1,float16,float16,0,1.0327040354410808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,1,128,1,float16,float16,0,0.46455466747283936
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,2,128,1,float16,float16,0,0.9497600396474203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,16,8,128,1,float16,float16,0,1.0705920060475667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,16,128,1,float16,float16,0,0.566271980603536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,2,128,1,float16,float16,0,0.4858880043029785
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,16,2,128,1,float16,float16,0,7.234901428222656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,4,128,1,float16,float16,0,0.544597347577413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,16,128,1,float16,float16,0,0.306005338827769
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,16,8,128,1,float16,float16,0,0.5452800194422404
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,2,128,1,float16,float16,0,0.272213339805603
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,8,128,1,float16,float16,0,0.29713066418965656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,1,128,1,float16,float16,0,0.2590720057487488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,16,4,128,1,float16,float16,0,0.29337600866953534
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,1,128,1,float16,float16,0,0.14711466431617737
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,2,128,1,float16,float16,0,0.15121066570281982
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,4,128,1,float16,float16,0,0.15923200050989786
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,8,128,1,float16,float16,0,0.1713493267695109
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,16,16,128,1,float16,float16,0,0.1807360053062439
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,16,128,1,float16,float16,0,2.606933275858561
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,1,128,1,float16,float16,0,1.9988479614257812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,2,128,1,float16,float16,0,2.1638827323913574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,4,128,1,float16,float16,0,2.4900266329447427
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,16,1,128,1,float16,float16,0,4.059136072794597
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,16,4,128,1,float16,float16,0,4.970154762268066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,16,128,1,float16,float16,0,1.317034641901652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,16,8,128,1,float16,float16,0,5.06385072072347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,1,128,1,float16,float16,0,1.0253653526306152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,16,8,128,1,float16,float16,0,2.4917333920796714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,16,128,1,float16,float16,0,0.6785706679026285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,2,128,1,float16,float16,0,1.1062613328297932
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,1,128,1,float16,float16,0,0.5439146757125854
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,2,128,1,float16,float16,0,0.5693440039952596
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,8,128,1,float16,float16,0,1.3042346636454265
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,4,128,1,float16,float16,0,0.6437546809514364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,16,4,128,1,float16,float16,0,1.2770986557006836
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,16,8,128,1,float16,float16,0,0.6493866840998331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,16,128,1,float16,float16,0,0.3633493185043335
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,1,128,1,float16,float16,0,0.2853546738624573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,2,128,1,float16,float16,0,0.3092479904492696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,4,128,1,float16,float16,0,0.3428693215052287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,16,128,1,float16,float16,0,0.19950934251149496
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,16,8,128,1,float16,float16,0,0.35225598017374676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,2,128,1,float16,float16,0,0.16776533921559653
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,1,128,1,float16,float16,0,0.15957333644231161
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,4,128,1,float16,float16,0,0.19438934326171875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,16,128,1,float16,float16,0,0.11963733037312825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,16,8,128,1,float16,float16,0,0.19268266359965006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,1,128,1,float16,float16,0,0.09830400347709656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,16,2,128,1,float16,float16,0,4.276565233866374
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,2,128,1,float16,float16,0,0.10035199920336406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,8,128,1,float16,float16,0,0.10922666390736897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,16,4,128,1,float16,float16,0,0.10769066214561462
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,16,128,1,float16,float16,0,2.7659947077433267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,1,128,1,float16,float16,0,1.9966293970743816
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,2,128,1,float16,float16,0,2.17958402633667
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,16,1,128,1,float16,float16,0,3.974656105041504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,16,2,128,1,float16,float16,0,4.299263954162598
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,16,8,128,1,float16,float16,0,5.2766720453898115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,8,128,1,float16,float16,0,2.6419199307759604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,16,4,128,1,float16,float16,0,2.5859413146972656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,1,128,1,float16,float16,0,1.0345813433329265
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,16,128,1,float16,float16,0,1.4035627047220867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,16,4,128,1,float16,float16,0,5.1498667399088545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,2,128,1,float16,float16,0,1.1221333344777424
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,1,128,1,float16,float16,0,0.5280426740646362
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,8,128,1,float16,float16,0,1.346218744913737
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,16,4,128,1,float16,float16,0,1.3240319887797039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,2,128,1,float16,float16,0,0.5642240047454834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,1,128,1,float16,float16,0,0.27135999997456867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,4,128,1,float16,float16,0,0.35037867228190106
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,16,128,1,float16,float16,0,0.37768534819285077
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,8,128,1,float16,float16,0,0.6898346741994222
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,2,128,1,float16,float16,0,0.16725333531697592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,2,128,1,float16,float16,0,0.2974720001220703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,16,128,1,float16,float16,0,0.2032639980316162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,1,128,1,float16,float16,0,0.15308800339698792
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,4,128,1,float16,float16,0,0.18773333231608072
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,16,8,128,1,float16,float16,0,0.1945599913597107
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,16,8,128,1,float16,float16,0,0.35498666763305664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,16,128,1,float16,float16,0,0.7142399946848551
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,1,128,1,float16,float16,0,0.0865280032157898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,2,128,1,float16,float16,0,0.09147733449935913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,4,128,1,float16,float16,0,0.10342400272687276
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,16,128,1,float16,float16,0,0.11758933464686076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,16,8,128,1,float16,float16,0,0.11076266566912334
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,1,128,1,float16,float16,0,0.05461333195368449
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,16,128,1,float16,float16,0,0.06519466638565063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,2,128,1,float16,float16,0,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,8,128,1,float16,float16,0,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,16,4,128,1,float16,float16,0,0.060415998101234436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,16,4,128,1,float16,float16,0,0.6722559928894043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,16,128,1,float16,float16,0,2.103125254313151
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,1,128,1,float16,float16,0,1.305087963740031
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,2,128,1,float16,float16,0,1.4958933194478352
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,4,128,1,float16,float16,0,1.917952060699463
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,16,128,1,float16,float16,0,1.0605226357777913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,16,1,128,1,float16,float16,0,2.5816747347513833
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,16,2,128,1,float16,float16,0,2.9057706197102866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,1,128,1,float16,float16,0,0.6599680185317993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,16,4,128,1,float16,float16,0,3.8051840464274087
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,2,128,1,float16,float16,0,0.7657813231150309
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,1,128,1,float16,float16,0,0.33826132615407306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,4,128,1,float16,float16,0,0.9710933367411295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,16,8,128,1,float16,float16,0,1.9505492846171062
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,16,8,128,1,float16,float16,0,0.9855999946594238
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,16,128,1,float16,float16,0,0.5422079960505167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,4,128,1,float16,float16,0,0.49510399500528973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,4,128,1,float16,float16,0,0.2582186659177144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,8,128,1,float16,float16,0,0.2657279968261719
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,16,128,1,float16,float16,0,0.283135990301768
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,8,128,1,float16,float16,0,0.5075626770655314
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,16,128,1,float16,float16,0,0.1527466674645742
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,1,128,1,float16,float16,0,0.10171733299891154
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,2,128,1,float16,float16,0,0.11383466919263203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,8,128,1,float16,float16,0,0.14353066682815552
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,16,128,1,float16,float16,0,0.08772266904513042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,1,128,1,float16,float16,0,0.05734399954477946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,2,128,1,float16,float16,0,0.20787199338277182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,16,4,128,1,float16,float16,0,0.13943466544151306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,4,128,1,float16,float16,0,0.07287466526031494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,2,128,1,float16,float16,0,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,16,8,128,1,float16,float16,0,0.08021333316961925
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,16,128,1,float16,float16,0,0.04761599997679392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,4,128,1,float16,float16,0,0.044031997521718345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,2,128,1,float16,float16,0,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,1,128,1,float16,float16,0,0.03908266623814901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,16,128,1,float16,float16,0,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,16,8,128,1,float16,float16,0,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,2,128,1,float16,float16,0,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,16,2,128,1,float16,float16,0,0.39048532644907635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,1,128,1,float16,float16,0,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,8,128,1,float16,float16,0,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,16,1,128,1,float16,float16,0,0.1802240014076233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,16,4,128,1,float16,float16,0,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,16,8,128,1,float16,float16,0,3.8942718505859375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,1,128,1,float16,float16,0,0.48418132464090985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,16,1,128,1,float16,float16,0,0.9709226290384928
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,16,2,128,1,float16,float16,0,1.1426133314768474
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,4,128,1,float16,float16,0,0.7917226950327555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,2,128,1,float16,float16,0,0.5826559861501058
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,16,128,1,float16,float16,0,0.8833706378936768
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,16,4,128,1,float16,float16,0,1.572864055633545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,16,8,128,1,float16,float16,0,0.8203946749369303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,16,8,128,1,float16,float16,0,1.60153595606486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,4,128,1,float16,float16,0,0.40004265308380127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,2,128,1,float16,float16,0,0.29713066418965656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,1,128,1,float16,float16,0,0.2445653279622396
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,8,128,1,float16,float16,0,0.21760000785191855
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,4,128,1,float16,float16,0,0.20889600118001303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,16,128,1,float16,float16,0,0.1264639993508657
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,16,128,1,float16,float16,0,0.44919466972351074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,2,128,1,float16,float16,0,0.08721066514650981
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,1,128,1,float16,float16,0,0.07458133498827617
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,1,128,1,float16,float16,0,0.13124266266822815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,4,128,1,float16,float16,0,0.11178666353225708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,16,128,1,float16,float16,0,0.0718506673971812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,1,128,1,float16,float16,0,0.04181333382924398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,2,128,1,float16,float16,0,0.04505600035190582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,4,128,1,float16,float16,0,0.055125330885251365
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,16,128,1,float16,float16,0,0.23534933725992838
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,16,128,1,float16,float16,0,0.03566933423280716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,16,8,128,1,float16,float16,0,0.06519466638565063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,16,8,128,1,float16,float16,0,0.4145493507385254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,1,128,1,float16,float16,0,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,8,128,1,float16,float16,0,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,2,128,1,float16,float16,0,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,16,128,1,float16,float16,0,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,2,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,8,128,1,float16,float16,0,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,4,128,1,float16,float16,0,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,1,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,16,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,2,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,4,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,16,8,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,16,4,128,1,float16,float16,0,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,16,8,128,1,float16,float16,0,0.1160533328851064
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,16,1,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,1,128,1,float16,float16,0,0.19268266359965006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,16,2,128,1,float16,float16,0,0.15684266885121664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,2,128,1,float16,float16,0,0.2474666635195414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,4,128,1,float16,float16,0,0.3548159996668498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,16,1,128,1,float16,float16,0,0.43263999621073407
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,16,128,1,float16,float16,0,0.4253013531366984
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,16,2,128,1,float16,float16,0,0.539136012395223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,16,4,128,1,float16,float16,0,0.7454720338185629
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,1,128,1,float16,float16,0,0.10342400272687276
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,16,128,1,float16,float16,0,0.21640533208847046
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,2,128,1,float16,float16,0,0.12919466694196066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,16,8,128,1,float16,float16,0,0.3735893170038859
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,16,8,128,1,float16,float16,0,0.7678293387095133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,1,128,1,float16,float16,0,0.06007466713587443
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,4,128,1,float16,float16,0,0.09745066364606221
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,1,128,1,float16,float16,0,0.03242666771014532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,2,128,1,float16,float16,0,0.07321600119272868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,2,128,1,float16,float16,0,0.03618133316437403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,4,128,1,float16,float16,0,0.04642133414745331
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,8,128,1,float16,float16,0,0.19319466749827066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,16,4,128,1,float16,float16,0,0.18312533696492514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,8,128,1,float16,float16,0,0.05819733440876007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,16,128,1,float16,float16,0,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,8,128,1,float16,float16,0,0.10342400272687276
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,1,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,16,16,128,1,float16,float16,0,0.116565336783727
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,16,128,1,float16,float16,0,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,2,128,1,float16,float16,0,0.01757866640885671
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,4,128,1,float16,float16,0,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,2,128,1,float16,float16,0,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,16,8,128,1,float16,float16,0,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,1,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,16,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,8,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,2,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,4,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,16,4,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,16,8,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,16,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,16,16,128,1,float16,float16,0,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,4,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,16,8,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,1,128,1,float16,float16,0,0.10444800059000652
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,2,128,1,float16,float16,0,0.13004799683888754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,4,128,1,float16,float16,0,0.18278400103251138
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,16,128,1,float16,float16,0,0.11520000298817952
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,16,2,128,1,float16,float16,0,0.244053324063619
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,1,128,1,float16,float16,0,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,16,128,1,float16,float16,0,0.21572266022364298
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,16,8,128,1,float16,float16,0,0.18995199600855509
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,16,4,128,1,float16,float16,0,0.3510613441467285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,16,8,128,1,float16,float16,0,0.36351998647054035
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,16,1,128,1,float16,float16,0,0.18926932414372763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,2,128,1,float16,float16,0,0.0730453332265218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,4,128,1,float16,float16,0,0.09762133161226909
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,8,128,1,float16,float16,0,0.05597866574923197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,16,128,1,float16,float16,0,0.06451199948787689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,16,8,128,1,float16,float16,0,0.1032533347606659
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,16,128,1,float16,float16,0,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,1,128,1,float16,float16,0,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,1,128,1,float16,float16,0,0.0315733328461647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,4,128,1,float16,float16,0,0.045738667249679565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,16,2,128,1,float16,float16,0,0.03566933423280716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,16,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,8,128,1,float16,float16,0,0.028501334289709728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,1,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,2,128,1,float16,float16,0,0.017407999684413273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,4,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,4,128,1,float16,float16,0,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,16,2,128,1,float16,float16,0,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,16,8,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,2,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,16,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,4,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,16,8,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,16,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,16,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,16,8,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,16,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,1,128,1,float16,float16,0,0.09147733449935913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,2,128,1,float16,float16,0,0.10376532872517903
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,16,128,1,float16,float16,0,0.13960533340771994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,16,1,128,1,float16,float16,0,0.16810667514801025
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,16,2,128,1,float16,float16,0,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,16,128,1,float16,float16,0,0.0769706666469574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,16,4,128,1,float16,float16,0,0.24337067206700644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,4,128,1,float16,float16,0,0.12714667121569315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,16,8,128,1,float16,float16,0,0.13124266266822815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,16,8,128,1,float16,float16,0,0.2486613392829895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,1,128,1,float16,float16,0,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,2,128,1,float16,float16,0,0.04966400067011515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,16,128,1,float16,float16,0,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,4,128,1,float16,float16,0,0.06092800199985504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,16,8,128,1,float16,float16,0,0.06997333467006683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,1,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,4,128,1,float16,float16,0,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,16,128,1,float16,float16,0,0.023552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,8,128,1,float16,float16,0,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,1,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,16,2,128,1,float16,float16,0,0.029525332152843475
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,2,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,4,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,16,8,128,1,float16,float16,0,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,16,128,1,float16,float16,0,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,2,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,4,128,1,float16,float16,0,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,8,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,16,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,16,1,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,16,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,16,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,16,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,16,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,16,8,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,1,128,1,float16,float16,0,0.07714133461316426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,16,128,1,float16,float16,0,0.10905599594116211
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,2,128,1,float16,float16,0,0.08089600006739299
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,4,128,1,float16,float16,0,0.09659733374913533
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,16,128,1,float16,float16,0,0.053930665055910744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,16,2,128,1,float16,float16,0,0.16810667514801025
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,1,128,1,float16,float16,0,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,16,1,128,1,float16,float16,0,0.15615999698638916
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,16,4,128,1,float16,float16,0,0.19353600343068442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,16,8,128,1,float16,float16,0,0.10342400272687276
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,2,128,1,float16,float16,0,0.04471466441949209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,16,8,128,1,float16,float16,0,0.19848533471425375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,16,128,1,float16,float16,0,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,1,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,4,128,1,float16,float16,0,0.048810665806134544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,2,128,1,float16,float16,0,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,16,8,128,1,float16,float16,0,0.04915200173854828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,4,128,1,float16,float16,0,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,16,8,128,1,float16,float16,0,0.029866665601730347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,1,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,8,128,1,float16,float16,0,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,4,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,1,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,16,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,2,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,16,16,128,1,float16,float16,0,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,4,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,16,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,16,8,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,16,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,16,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,16,16,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,16,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,0,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,0,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,16,128,1,float16,float16,0,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,0,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,0,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,0,0.05376000205675761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,0,0.08413867155710857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,16,128,1,float16,float16,0,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,0,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,0,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,0,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,16,128,1,float16,float16,0,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,0,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,0,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,16,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,16,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,16,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,16,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,16,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,12,12,128,1,float16,float16,0,9.493674596150717
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,12,1,128,1,float16,float16,0,17.947818756103516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,12,2,128,1,float16,float16,0,18.212181091308594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,12,4,128,1,float16,float16,0,18.80234654744466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,12,12,128,1,float16,float16,0,19.082922617594402
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,12,1,128,1,float16,float16,0,8.98082160949707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,12,12,128,1,float16,float16,0,4.746239980061849
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,12,1,128,1,float16,float16,0,4.507818539937337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,12,2,128,1,float16,float16,0,4.704256057739258
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,12,2,128,1,float16,float16,0,8.977578481038412
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,12,4,128,1,float16,float16,0,9.220266977945963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,12,4,128,1,float16,float16,0,4.802389462788899
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,12,1,128,1,float16,float16,0,36.18167368570963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,12,2,128,1,float16,float16,0,36.89574432373047
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,12,4,128,1,float16,float16,0,37.48949432373047
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,12,12,128,1,float16,float16,0,10.801663716634115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,12,12,128,1,float16,float16,0,5.488298416137695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,12,1,128,1,float16,float16,0,10.327893575032553
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,12,2,128,1,float16,float16,0,10.38916269938151
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,12,4,128,1,float16,float16,0,10.871808369954428
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,12,1,128,1,float16,float16,0,5.083818753560384
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,12,2,128,1,float16,float16,0,5.2391252517700195
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,12,4,128,1,float16,float16,0,5.402111689249675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,12,1,128,1,float16,float16,0,20.56942876180013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,12,2,128,1,float16,float16,0,20.92356236775716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,12,12,128,1,float16,float16,0,2.9016745885213218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,12,1,128,1,float16,float16,0,2.69704532623291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,12,2,128,1,float16,float16,0,2.686463991800944
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,12,4,128,1,float16,float16,0,21.155328114827473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,12,4,128,1,float16,float16,0,2.7854506174723306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,12,12,128,1,float16,float16,0,3.8161067962646484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,12,12,128,1,float16,float16,0,7.624703725179036
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,12,1,128,1,float16,float16,0,7.3270613352457685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,12,2,128,1,float16,float16,0,7.111167907714844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,12,4,128,1,float16,float16,0,7.41649055480957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,12,1,128,1,float16,float16,0,3.628373463948568
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,12,12,128,1,float16,float16,0,2.0497066179911294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,12,2,128,1,float16,float16,0,3.67906125386556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,12,4,128,1,float16,float16,0,3.772245407104492
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,12,1,128,1,float16,float16,0,14.600362141927084
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,12,2,128,1,float16,float16,0,14.702592213948568
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,12,4,128,1,float16,float16,0,15.263402303059896
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,12,2,128,1,float16,float16,0,1.895253340403239
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,12,1,128,1,float16,float16,0,1.8781867027282715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,12,4,128,1,float16,float16,0,2.00601593653361
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,12,1,128,1,float16,float16,0,9.251498540242514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,12,2,128,1,float16,float16,0,9.550336201985678
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,12,12,128,1,float16,float16,0,10.065237045288086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,12,4,128,1,float16,float16,0,9.948160171508789
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,12,12,128,1,float16,float16,0,2.5482239723205566
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,12,2,128,1,float16,float16,0,4.779520034790039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,12,12,128,1,float16,float16,0,4.935680071512858
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,12,1,128,1,float16,float16,0,4.436821301778157
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,12,1,128,1,float16,float16,0,18.378922780354817
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,12,4,128,1,float16,float16,0,4.87662919362386
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,12,1,128,1,float16,float16,0,2.334890683492025
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,12,2,128,1,float16,float16,0,19.170475006103516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,12,4,128,1,float16,float16,0,19.99223454793294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,12,4,128,1,float16,float16,0,1.3276159763336182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,12,2,128,1,float16,float16,0,1.2779520352681477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,12,1,128,1,float16,float16,0,1.2513279914855957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,12,4,128,1,float16,float16,0,2.486442724863688
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,12,2,128,1,float16,float16,0,2.4596479733784995
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,12,12,128,1,float16,float16,0,1.3281280199686687
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,12,12,128,1,float16,float16,0,2.885631879170736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,12,1,128,1,float16,float16,0,5.137749354044597
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,12,2,128,1,float16,float16,0,5.323605219523112
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,12,12,128,1,float16,float16,0,5.786965052286784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,12,4,128,1,float16,float16,0,5.667839686075847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,12,1,128,1,float16,float16,0,2.5966933568318686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,12,1,128,1,float16,float16,0,10.866004943847656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,12,12,128,1,float16,float16,0,1.5778133074442546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,12,1,128,1,float16,float16,0,1.3902506828308105
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,12,2,128,1,float16,float16,0,2.7327146530151367
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,12,4,128,1,float16,float16,0,2.873173395792643
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,12,2,128,1,float16,float16,0,10.87112553914388
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,12,1,128,1,float16,float16,0,0.7543466885884603
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,12,12,128,1,float16,float16,0,0.8308053016662598
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,12,2,128,1,float16,float16,0,1.3994666735331218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,12,4,128,1,float16,float16,0,1.525760014851888
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,12,2,128,1,float16,float16,0,0.7702186902364095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,12,4,128,1,float16,float16,0,0.807253360748291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,12,4,128,1,float16,float16,0,11.638954162597656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,12,12,128,1,float16,float16,0,2.8064425786336265
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,12,2,128,1,float16,float16,0,4.913663864135742
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,12,1,128,1,float16,float16,0,4.800511995951335
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,12,12,128,1,float16,float16,0,5.460821151733398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,12,4,128,1,float16,float16,0,5.302101453145345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,12,1,128,1,float16,float16,0,2.3922346433003745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,12,12,128,1,float16,float16,0,1.460053284962972
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,12,2,128,1,float16,float16,0,2.520576000213623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,12,1,128,1,float16,float16,0,1.2386986414591472
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,12,4,128,1,float16,float16,0,2.740053176879883
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,12,2,128,1,float16,float16,0,1.302016019821167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,12,1,128,1,float16,float16,0,0.6638933420181274
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,12,12,128,1,float16,float16,0,0.764245351155599
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,12,2,128,1,float16,float16,0,10.158079783121744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,12,2,128,1,float16,float16,0,0.6893226305643717
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,12,1,128,1,float16,float16,0,9.829205195109049
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,12,4,128,1,float16,float16,0,1.3849600156148274
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,12,1,128,1,float16,float16,0,0.373418649037679
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,12,12,128,1,float16,float16,0,0.41727999846140545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,12,4,128,1,float16,float16,0,0.7371093432108561
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,12,2,128,1,float16,float16,0,0.40857601165771484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,12,4,128,1,float16,float16,0,0.4092586835225423
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,12,4,128,1,float16,float16,0,10.956287384033203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,12,12,128,1,float16,float16,0,1.7061546643575032
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,12,2,128,1,float16,float16,0,2.8989439010620117
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,12,1,128,1,float16,float16,0,2.7448320388793945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,12,12,128,1,float16,float16,0,3.3075199127197266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,12,4,128,1,float16,float16,0,3.2128000259399414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,12,1,128,1,float16,float16,0,1.4015146891276042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,12,12,128,1,float16,float16,0,0.8710827032725016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,12,1,128,1,float16,float16,0,0.7323306401570638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,12,1,128,1,float16,float16,0,5.570218404134114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,12,2,128,1,float16,float16,0,1.4822400410970051
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,12,2,128,1,float16,float16,0,5.814613342285156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,12,4,128,1,float16,float16,0,1.6257707277933757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,12,2,128,1,float16,float16,0,0.772437334060669
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,12,12,128,1,float16,float16,0,0.47086934248606366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,12,2,128,1,float16,float16,0,0.4225706656773885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,12,4,128,1,float16,float16,0,6.347946802775065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,12,4,128,1,float16,float16,0,0.8425813515981039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,12,12,128,1,float16,float16,0,0.27084799607594806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,12,4,128,1,float16,float16,0,0.48622934023539227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,12,1,128,1,float16,float16,0,0.42871467272440594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,12,2,128,1,float16,float16,0,0.23603200912475586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,12,1,128,1,float16,float16,0,0.23654399315516153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,12,4,128,1,float16,float16,0,0.26077866554260254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,12,12,128,1,float16,float16,0,1.6940372784932454
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,12,1,128,1,float16,float16,0,2.5792853037516275
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,12,12,128,1,float16,float16,0,3.34114138285319
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,12,4,128,1,float16,float16,0,3.2025601069132485
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,12,2,128,1,float16,float16,0,2.761727968851725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,12,1,128,1,float16,float16,0,1.3090133666992188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,12,2,128,1,float16,float16,0,1.4445226987202961
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,12,1,128,1,float16,float16,0,0.6736213366190592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,12,12,128,1,float16,float16,0,0.8686933517456055
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,12,4,128,1,float16,float16,0,1.6276480356852214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,12,1,128,1,float16,float16,0,5.142186800638835
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,12,4,128,1,float16,float16,0,6.424234390258789
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,12,2,128,1,float16,float16,0,0.7383039792378744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,12,2,128,1,float16,float16,0,5.540351867675781
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,12,12,128,1,float16,float16,0,0.4580693244934082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,12,1,128,1,float16,float16,0,0.3633493185043335
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,12,4,128,1,float16,float16,0,0.8345599969228109
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,12,4,128,1,float16,float16,0,0.45926400025685626
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,12,2,128,1,float16,float16,0,0.4102826515833537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,12,12,128,1,float16,float16,0,0.2524159948031108
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,12,2,128,1,float16,float16,0,0.21742933988571167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,12,12,128,1,float16,float16,0,0.15018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,12,4,128,1,float16,float16,0,0.2435413400332133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,12,1,128,1,float16,float16,0,0.1288533310095469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,12,2,128,1,float16,float16,0,0.13380266229311624
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,12,4,128,1,float16,float16,0,0.14045866330464682
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,12,1,128,1,float16,float16,0,0.20736000935236612
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,12,1,128,1,float16,float16,0,1.548799991607666
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,12,12,128,1,float16,float16,0,2.1232639948527017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,12,4,128,1,float16,float16,0,2.011136054992676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,12,1,128,1,float16,float16,0,3.0568106969197593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,12,4,128,1,float16,float16,0,3.9765332539876304
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,12,12,128,1,float16,float16,0,1.0796373685201008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,12,1,128,1,float16,float16,0,0.7895039717356364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,12,2,128,1,float16,float16,0,1.6798720359802246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,12,2,128,1,float16,float16,0,3.3023999532063804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,12,2,128,1,float16,float16,0,0.8739840189615885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,12,1,128,1,float16,float16,0,0.4128426710764567
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,12,12,128,1,float16,float16,0,0.5596160093943278
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,12,2,128,1,float16,float16,0,0.45772798856099445
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,12,4,128,1,float16,float16,0,1.0408960183461506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,12,2,128,1,float16,float16,0,0.2469546596209208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,12,4,128,1,float16,float16,0,0.28433066606521606
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,12,12,128,1,float16,float16,0,0.16725333531697592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,12,1,128,1,float16,float16,0,0.22613332668940225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,12,4,128,1,float16,float16,0,0.5275306701660156
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,12,1,128,1,float16,float16,0,0.12919466694196066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,12,2,128,1,float16,float16,0,0.13704533378283182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,12,12,128,1,float16,float16,0,0.09983999530474345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,12,12,128,1,float16,float16,0,0.29781333605448407
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,12,1,128,1,float16,float16,0,0.08055466910203297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,12,2,128,1,float16,float16,0,0.08533333738644917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,12,4,128,1,float16,float16,0,0.16145066420237222
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,12,4,128,1,float16,float16,0,0.09403733412424724
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,12,12,128,1,float16,float16,0,2.2775467236836753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,12,1,128,1,float16,float16,0,1.5136426289876301
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,12,12,128,1,float16,float16,0,1.152341365814209
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,12,2,128,1,float16,float16,0,0.874666690826416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,12,1,128,1,float16,float16,0,0.7712426980336508
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,12,2,128,1,float16,float16,0,1.7162240346272786
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,12,4,128,1,float16,float16,0,2.1534719467163086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,12,1,128,1,float16,float16,0,2.997589429219564
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,12,1,128,1,float16,float16,0,0.3998719851175944
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,12,12,128,1,float16,float16,0,0.589141329129537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,12,2,128,1,float16,float16,0,0.4505600134531657
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,12,1,128,1,float16,float16,0,0.2135039965311686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,12,2,128,1,float16,float16,0,3.3582080205281577
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,12,4,128,1,float16,float16,0,0.2908160090446472
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,12,2,128,1,float16,float16,0,0.24132267634073892
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,12,12,128,1,float16,float16,0,0.1704960068066915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,12,4,128,1,float16,float16,0,0.1634986698627472
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,12,1,128,1,float16,float16,0,0.12134400010108948
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,12,12,128,1,float16,float16,0,0.3155626654624939
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,12,4,128,1,float16,float16,0,4.245162645975749
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,12,12,128,1,float16,float16,0,0.1013759970664978
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,12,2,128,1,float16,float16,0,0.07645866771539052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,12,4,128,1,float16,float16,0,0.08635733524958293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,12,4,128,1,float16,float16,0,1.0816853046417236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,12,1,128,1,float16,float16,0,0.07423999905586243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,12,4,128,1,float16,float16,0,0.553984006245931
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,12,1,128,1,float16,float16,0,0.04505600035190582
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,12,2,128,1,float16,float16,0,0.04693333307902018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,12,4,128,1,float16,float16,0,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,12,2,128,1,float16,float16,0,0.13482667009035745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,12,12,128,1,float16,float16,0,1.7641812960306804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,12,1,128,1,float16,float16,0,1.014954646428426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,12,12,128,1,float16,float16,0,0.051370665431022644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,12,2,128,1,float16,float16,0,1.2243626912434895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,12,1,128,1,float16,float16,0,1.999872048695882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,12,12,128,1,float16,float16,0,0.8959999879201254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,12,1,128,1,float16,float16,0,0.5172906716664633
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,12,2,128,1,float16,float16,0,0.6171306769053141
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,12,4,128,1,float16,float16,0,0.8265386422475179
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,12,4,128,1,float16,float16,0,1.626794656117757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,12,1,128,1,float16,float16,0,0.2672640085220337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,12,12,128,1,float16,float16,0,0.457045316696167
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,12,2,128,1,float16,float16,0,0.31829333305358887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,12,4,128,1,float16,float16,0,0.42342400550842285
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,12,2,128,1,float16,float16,0,0.17117865880330405
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,12,2,128,1,float16,float16,0,2.3567360242207847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,12,12,128,1,float16,float16,0,0.13107200463612875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,12,4,128,1,float16,float16,0,0.22169599930445352
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,12,12,128,1,float16,float16,0,0.24251733223597208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,12,2,128,1,float16,float16,0,0.09454933802286784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,12,1,128,1,float16,float16,0,0.0817493349313736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,12,2,128,1,float16,float16,0,0.053077335158983864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,12,4,128,1,float16,float16,0,0.12185600399971008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,12,1,128,1,float16,float16,0,0.04966400067011515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,12,4,128,1,float16,float16,0,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,12,1,128,1,float16,float16,0,0.14404267072677612
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,12,12,128,1,float16,float16,0,0.03566933423280716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,12,4,128,1,float16,float16,0,3.219967842102051
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,12,1,128,1,float16,float16,0,0.0290133332212766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,12,2,128,1,float16,float16,0,0.03054933249950409
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,12,4,128,1,float16,float16,0,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,12,12,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,12,12,128,1,float16,float16,0,0.07509333391984303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,12,1,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,12,4,128,1,float16,float16,0,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,12,1,128,1,float16,float16,0,0.3853653271993001
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,12,2,128,1,float16,float16,0,0.4877653519312541
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,12,2,128,1,float16,float16,0,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,12,4,128,1,float16,float16,0,0.6927359898885092
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,12,1,128,1,float16,float16,0,0.1945599913597107
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,12,12,128,1,float16,float16,0,0.38997332255045575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,12,1,128,1,float16,float16,0,0.7601493199666342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,12,2,128,1,float16,float16,0,0.9557332992553711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,12,2,128,1,float16,float16,0,0.2515626748402913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,12,4,128,1,float16,float16,0,1.3760852813720703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,12,12,128,1,float16,float16,0,0.20428800582885742
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,12,12,128,1,float16,float16,0,0.11161599556605022
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,12,12,128,1,float16,float16,0,0.7610026995340983
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,12,4,128,1,float16,float16,0,0.09949866930643718
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,12,4,128,1,float16,float16,0,0.3543039957682292
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,12,1,128,1,float16,float16,0,0.034645333886146545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,12,2,128,1,float16,float16,0,0.07389866809050243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,12,4,128,1,float16,float16,0,0.046762665112813316
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,12,12,128,1,float16,float16,0,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,12,1,128,1,float16,float16,0,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,12,1,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,12,2,128,1,float16,float16,0,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,12,12,128,1,float16,float16,0,0.020309332758188248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,12,1,128,1,float16,float16,0,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,12,2,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,12,2,128,1,float16,float16,0,0.13329066832860312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,12,4,128,1,float16,float16,0,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,12,2,128,1,float16,float16,0,0.03908266623814901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,12,4,128,1,float16,float16,0,0.18449066082636514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,12,4,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,12,12,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,12,2,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,12,4,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,12,1,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,12,1,128,1,float16,float16,0,0.059392000238100685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,12,1,128,1,float16,float16,0,0.15633066495259604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,12,2,128,1,float16,float16,0,0.21538132429122925
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,12,12,128,1,float16,float16,0,0.061610668897628784
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,12,1,128,1,float16,float16,0,0.33928533395131427
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,12,12,128,1,float16,float16,0,0.1889280080795288
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,12,4,128,1,float16,float16,0,0.3218773404757182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,12,1,128,1,float16,float16,0,0.08618666728337605
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,12,2,128,1,float16,float16,0,0.11349333326021831
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,12,12,128,1,float16,float16,0,0.36693334579467773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,12,2,128,1,float16,float16,0,0.44202665487925213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,12,12,128,1,float16,float16,0,0.1032533347606659
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,12,4,128,1,float16,float16,0,0.16708266735076904
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,12,2,128,1,float16,float16,0,0.062463998794555664
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,12,4,128,1,float16,float16,0,0.6582613388697306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,12,4,128,1,float16,float16,0,0.08994133273760478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,12,12,128,1,float16,float16,0,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,12,2,128,1,float16,float16,0,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,12,1,128,1,float16,float16,0,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,12,4,128,1,float16,float16,0,0.040106666584809623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,12,2,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,12,4,128,1,float16,float16,0,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,12,12,128,1,float16,float16,0,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,12,2,128,1,float16,float16,0,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,12,1,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,12,4,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,12,12,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,12,2,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,12,4,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,12,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,12,12,128,1,float16,float16,0,0.05563733478387197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,12,1,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,12,12,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,12,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,12,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,12,4,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,12,1,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,12,12,128,1,float16,float16,0,0.10274133086204529
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,12,1,128,1,float16,float16,0,0.08584533135096233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,12,4,128,1,float16,float16,0,0.16554666558901468
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,12,2,128,1,float16,float16,0,0.11127466956774394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,12,12,128,1,float16,float16,0,0.18824533621470133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,12,1,128,1,float16,float16,0,0.04761599997679392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,12,2,128,1,float16,float16,0,0.0631466656923294
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,12,12,128,1,float16,float16,0,0.05495466788609823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,12,2,128,1,float16,float16,0,0.20940800507863364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,12,2,128,1,float16,float16,0,0.03089066594839096
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,12,4,128,1,float16,float16,0,0.08942932883898418
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,12,4,128,1,float16,float16,0,0.31436800956726074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,12,12,128,1,float16,float16,0,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,12,4,128,1,float16,float16,0,0.03925333420435587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,12,1,128,1,float16,float16,0,0.15547733505566916
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,12,1,128,1,float16,float16,0,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,12,1,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,12,12,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,12,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,12,4,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,12,2,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,12,12,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,12,2,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,12,2,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,12,12,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,12,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,12,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,12,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,12,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,12,4,128,1,float16,float16,0,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,12,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,12,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,12,12,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,12,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,12,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,12,2,128,1,float16,float16,0,0.08567466338475545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,12,4,128,1,float16,float16,0,0.11127466956774394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,12,1,128,1,float16,float16,0,0.07202133536338806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,12,12,128,1,float16,float16,0,0.12083199620246887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,12,1,128,1,float16,float16,0,0.13499733805656433
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,12,2,128,1,float16,float16,0,0.15889066457748413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,12,12,128,1,float16,float16,0,0.06638933221499126
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,12,2,128,1,float16,float16,0,0.04164266586303711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,12,1,128,1,float16,float16,0,0.038058665891488395
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,12,4,128,1,float16,float16,0,0.21009065707524618
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,12,12,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,12,2,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,12,4,128,1,float16,float16,0,0.05171200136343638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,12,4,128,1,float16,float16,0,0.02918400118748347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,12,12,128,1,float16,float16,0,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,12,1,128,1,float16,float16,0,0.02372266600529353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,12,1,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,12,4,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,12,2,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,12,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,12,12,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,12,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,12,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,12,12,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,12,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,12,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,12,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,12,12,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,12,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,12,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,12,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,12,12,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,12,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,12,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,12,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,12,2,128,1,float16,float16,0,0.06553600231806438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,12,1,128,1,float16,float16,0,0.06178133189678192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,12,12,128,1,float16,float16,0,0.09028266867001851
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,12,4,128,1,float16,float16,0,0.07526400188604991
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,12,12,128,1,float16,float16,0,0.04369066655635834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,12,12,128,1,float16,float16,0,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,12,1,128,1,float16,float16,0,0.12100266416867574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,12,2,128,1,float16,float16,0,0.13567999998728433
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,12,2,128,1,float16,float16,0,0.03788800040880839
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,12,1,128,1,float16,float16,0,0.03618133316437403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,12,1,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,12,4,128,1,float16,float16,0,0.041984001795450844
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,12,4,128,1,float16,float16,0,0.16059733430544534
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,12,1,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,12,12,128,1,float16,float16,0,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,12,2,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,12,12,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,12,2,128,1,float16,float16,0,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,12,4,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,12,4,128,1,float16,float16,0,0.025600001215934753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,12,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,12,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,12,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,12,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,12,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,12,12,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,12,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,12,12,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,12,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,12,2,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,12,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,12,12,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,12,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,12,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,12,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,12,12,128,1,float16,float16,0,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,12,12,128,1,float16,float16,0,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,0,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,12,12,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,0,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,0,0.05273599922657013
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,0,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,12,12,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,0,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,12,12,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,12,12,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,12,12,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,12,12,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,1,128,1,float16,float16,0,11.944447835286459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,2,128,1,float16,float16,0,12.290218353271484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,float16,0,12.815701802571615
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,4,128,1,float16,float16,0,12.954964955647787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,float16,0,6.356992085774739
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,1,128,1,float16,float16,0,6.17250124613444
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,2,128,1,float16,float16,0,6.221312204996745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,4,128,1,float16,float16,0,6.461952209472656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,float16,0,3.3522345225016275
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,1,128,1,float16,float16,0,3.2380587259928384
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,8,1,128,1,float16,float16,0,23.994710286458332
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,8,2,128,1,float16,float16,0,24.513707478841145
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,2,128,1,float16,float16,0,3.2368640899658203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,8,4,128,1,float16,float16,0,25.488555908203125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,float16,0,6.388736089070638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,4,128,1,float16,float16,0,3.3030827840169272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,float16,0,12.757845560709635
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,float16,0,3.3416531880696616
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,1,128,1,float16,float16,0,6.806186676025391
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,float16,0,7.345663706461589
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,2,128,1,float16,float16,0,7.033002853393555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,float16,0,7.422122955322266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,8,4,128,1,float16,float16,0,7.267157236735026
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,1,128,1,float16,float16,0,3.518122673034668
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,float16,0,3.708245277404785
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,8,1,128,1,float16,float16,0,13.679616292317709
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,8,2,128,1,float16,float16,0,14.093140920003256
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,2,128,1,float16,float16,0,3.5652265548706055
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,8,4,128,1,float16,float16,0,14.84970728556315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,4,128,1,float16,float16,0,3.7166080474853516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,float16,0,1.9730772972106934
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,float16,0,3.7457920710245767
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,1,128,1,float16,float16,0,1.833130677541097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,2,128,1,float16,float16,0,1.9083946545918782
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,4,128,1,float16,float16,0,2.0032854080200195
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,float16,0,1.9869012832641602
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,1,128,1,float16,float16,0,4.753919919331868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,2,128,1,float16,float16,0,4.85700257619222
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,float16,0,5.2111358642578125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,4,128,1,float16,float16,0,5.121365229288737
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,float16,0,5.213354746500651
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,float16,0,2.6794665654500327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,1,128,1,float16,float16,0,2.4430932998657227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,8,1,128,1,float16,float16,0,9.796607971191406
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,2,128,1,float16,float16,0,2.5569279988606772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,4,128,1,float16,float16,0,2.6659839948018393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,8,2,128,1,float16,float16,0,9.987071990966797
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,float16,0,2.687829335530599
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,1,128,1,float16,float16,0,1.3085013230641682
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,float16,0,1.430357297261556
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,8,4,128,1,float16,float16,0,10.385066350301107
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,2,128,1,float16,float16,0,1.3900799751281738
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,4,128,1,float16,float16,0,1.4235307375590007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,float16,0,1.4149972597757976
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,1,128,1,float16,float16,0,6.234453201293945
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,float16,0,6.765567779541016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,2,128,1,float16,float16,0,6.4996693929036455
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,4,128,1,float16,float16,0,6.812501271565755
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,float16,0,6.798847834269206
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,float16,0,3.5638612111409507
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,1,128,1,float16,float16,0,3.0837761561075845
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,2,128,1,float16,float16,0,3.36844793955485
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,4,128,1,float16,float16,0,3.431424140930176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,8,1,128,1,float16,float16,0,12.650836944580078
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,float16,0,3.4372266133626304
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,8,2,128,1,float16,float16,0,12.809727986653646
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,8,4,128,1,float16,float16,0,13.389141082763672
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,float16,0,0.9704106648763021
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,1,128,1,float16,float16,0,0.8939519723256429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,2,128,1,float16,float16,0,0.9265493551890055
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,1,128,1,float16,float16,0,1.6092160542805989
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,float16,0,1.7795413335164387
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,4,128,1,float16,float16,0,1.7677653630574544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,2,128,1,float16,float16,0,1.6640000343322754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,float16,0,1.9159040451049805
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,4,128,1,float16,float16,0,0.9755307038625082
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,float16,0,0.974677324295044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,1,128,1,float16,float16,0,3.4757973353068032
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,float16,0,3.9823360443115234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,2,128,1,float16,float16,0,3.597994804382324
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,4,128,1,float16,float16,0,3.9599787394205728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,8,1,128,1,float16,float16,0,7.05621337890625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,float16,0,2.0710399945576987
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,8,2,128,1,float16,float16,0,7.266986846923828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,1,128,1,float16,float16,0,1.8123092651367188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,2,128,1,float16,float16,0,1.8918399810791016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,8,4,128,1,float16,float16,0,7.932586669921875
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,1,128,1,float16,float16,0,0.9582933584849039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,float16,0,3.998719851175944
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,float16,0,1.1240106423695881
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,2,128,1,float16,float16,0,1.0031786759694417
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,4,128,1,float16,float16,0,2.0780372619628906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,4,128,1,float16,float16,0,1.1083093484242756
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,float16,0,2.063530604044596
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,1,128,1,float16,float16,0,0.5521066586176554
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,float16,0,0.6287360191345215
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,2,128,1,float16,float16,0,0.5584213336308798
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,float16,0,1.0806612968444824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,4,128,1,float16,float16,0,0.6157653331756592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,float16,0,0.6121813456217448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,1,128,1,float16,float16,0,3.183786710103353
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,2,128,1,float16,float16,0,3.345919926961263
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,float16,0,3.842048009236654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,4,128,1,float16,float16,0,3.803647994995117
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,1,128,1,float16,float16,0,1.6184320449829102
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,2,128,1,float16,float16,0,1.744383970896403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,float16,0,1.9913387298583984
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,8,1,128,1,float16,float16,0,6.655487696329753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,8,2,128,1,float16,float16,0,6.7723948160807295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,4,128,1,float16,float16,0,1.9379199345906575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,8,4,128,1,float16,float16,0,7.666858673095703
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,1,128,1,float16,float16,0,0.8591360251108805
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,float16,0,1.0292906761169434
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,4,128,1,float16,float16,0,1.0398720105489094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,float16,0,0.5693440039952596
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,float16,0,1.021781365076701
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,8,2,128,1,float16,float16,0,0.9253546396891276
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,1,128,1,float16,float16,0,0.4679679870605469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,2,128,1,float16,float16,0,0.49664000670115155
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,float16,0,3.8213974634806314
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,4,128,1,float16,float16,0,0.559445341428121
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,float16,0,1.9578879674275715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,float16,0,0.32358400026957196
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,4,128,1,float16,float16,0,0.31624533732732135
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,1,128,1,float16,float16,0,0.27562665939331055
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,float16,0,0.558079997698466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,2,128,1,float16,float16,0,0.2795520027478536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,float16,0,0.32392533620198566
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,2,128,1,float16,float16,0,2.0253012975056968
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,1,128,1,float16,float16,0,1.8602666854858398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,float16,0,2.3637332916259766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,8,2,128,1,float16,float16,0,3.9348907470703125
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,4,128,1,float16,float16,0,2.308095932006836
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,float16,0,2.355029265085856
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,8,1,128,1,float16,float16,0,3.7034667332967124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,1,128,1,float16,float16,0,0.9613653024037679
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,float16,0,1.2182186444600422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,2,128,1,float16,float16,0,1.0451626777648926
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,4,128,1,float16,float16,0,1.1847679615020752
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,float16,0,0.6389760176340739
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,float16,0,1.225215991338094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,8,4,128,1,float16,float16,0,4.576256116231282
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,4,128,1,float16,float16,0,0.6311253309249878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,float16,0,0.35037867228190106
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,2,128,1,float16,float16,0,0.3126613299051921
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,float16,0,0.6398293177286783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,4,128,1,float16,float16,0,0.3556693394978841
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,2,128,1,float16,float16,0,0.5507413148880005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,1,128,1,float16,float16,0,0.291157325108846
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,float16,0,0.20360533396402994
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,1,128,1,float16,float16,0,0.17015467087427774
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,8,1,128,1,float16,float16,0,0.5230933427810669
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,2,128,1,float16,float16,0,0.17407999436060587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,float16,0,0.35174401601155597
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,float16,0,0.20087466637293497
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,8,4,128,1,float16,float16,0,0.1914880077044169
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,float16,0,2.4396799405415854
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,1,128,1,float16,float16,0,1.7693012555440266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,4,128,1,float16,float16,0,2.370730717976888
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,float16,0,2.4197120666503906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,8,2,128,1,float16,float16,0,1.9478185971577961
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,1,128,1,float16,float16,0,0.9060693581899008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,float16,0,1.245184024175008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,2,128,1,float16,float16,0,1.0026666323343914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,4,128,1,float16,float16,0,1.2161706288655598
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,8,4,128,1,float16,float16,0,4.691455841064453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,float16,0,1.2364799976348877
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,float16,0,0.6446079810460409
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,8,2,128,1,float16,float16,0,3.7749760945638022
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,1,128,1,float16,float16,0,0.47223468621571857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,4,128,1,float16,float16,0,0.6260053316752116
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,8,1,128,1,float16,float16,0,3.501568158467611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,2,128,1,float16,float16,0,0.5230933427810669
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,float16,0,0.6418773333231608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,float16,0,0.3442346652348836
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,1,128,1,float16,float16,0,0.26282666126887005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,float16,0,0.34355199337005615
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,1,128,1,float16,float16,0,0.14830933014551798
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,2,128,1,float16,float16,0,0.15957333644231161
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,4,128,1,float16,float16,0,0.33894399801890057
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,4,128,1,float16,float16,0,0.19268266359965006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,1,128,1,float16,float16,0,0.09898666540781657
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,float16,0,0.1986560026804606
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,float16,0,0.1114453375339508
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,2,128,1,float16,float16,0,0.10086400310198466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,float16,0,0.20002132654190063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,4,128,1,float16,float16,0,0.11110400160153706
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,8,2,128,1,float16,float16,0,0.292522668838501
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,float16,0,0.11315199732780457
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,1,128,1,float16,float16,0,1.072640021642049
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,float16,0,1.5663785934448242
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,4,128,1,float16,float16,0,1.5290026664733887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,2,128,1,float16,float16,0,1.213098684946696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,8,4,128,1,float16,float16,0,3.0230185190836587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,8,1,128,1,float16,float16,0,2.092202663421631
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,float16,0,0.7978666623433431
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,1,128,1,float16,float16,0,0.5495466788609823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,2,128,1,float16,float16,0,0.6290773153305054
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,8,2,128,1,float16,float16,0,2.3352319399515786
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,float16,0,0.42137598991394043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,float16,0,1.5641600290934246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,1,128,1,float16,float16,0,0.29286400477091473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,4,128,1,float16,float16,0,0.784554640452067
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,2,128,1,float16,float16,0,0.33553067843119305
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,float16,0,0.814250628153483
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,float16,0,0.23125332593917847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,1,128,1,float16,float16,0,0.16708266735076904
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,float16,0,0.4225706656773885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,2,128,1,float16,float16,0,0.18926932414372763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,float16,0,0.2315946618715922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,8,4,128,1,float16,float16,0,0.22630399465560913
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,float16,0,0.13636266191800436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,1,128,1,float16,float16,0,0.09983999530474345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,8,4,128,1,float16,float16,0,0.4094293514887492
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,4,128,1,float16,float16,0,0.12185600399971008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,2,128,1,float16,float16,0,0.10615467031796773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,float16,0,0.06894933183987935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,float16,0,0.13550933202107748
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,1,128,1,float16,float16,0,0.058880001306533813
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,2,128,1,float16,float16,0,0.06178133189678192
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,4,128,1,float16,float16,0,0.0682666649421056
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,float16,0,0.06894933183987935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,float16,0,1.7082026799519856
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,1,128,1,float16,float16,0,1.0658133029937744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,2,128,1,float16,float16,0,1.2581546306610107
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,float16,0,0.8779093424479166
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,1,128,1,float16,float16,0,0.5466453234354655
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,4,128,1,float16,float16,0,1.6814079284667969
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,float16,0,1.70632537206014
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,2,128,1,float16,float16,0,0.6505813201268514
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,8,1,128,1,float16,float16,0,2.09663995107015
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,float16,0,0.8751786549886068
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,8,4,128,1,float16,float16,0,0.8574293454488119
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,8,2,128,1,float16,float16,0,2.4191999435424805
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,2,128,1,float16,float16,0,0.33655468622843426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,float16,0,0.45482667287190753
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,float16,0,0.45311999320983887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,4,128,1,float16,float16,0,0.44458667437235516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,8,4,128,1,float16,float16,0,3.3082027435302734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,8,1,128,1,float16,float16,0,0.28706133365631104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,2,128,1,float16,float16,0,0.18466132879257202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,float16,0,0.24371200799942017
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,4,128,1,float16,float16,0,0.23466666539510092
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,float16,0,0.2445653279622396
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,1,128,1,float16,float16,0,0.09130666653315227
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,float16,0,0.1378986636797587
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,float16,0,0.1384106675783793
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,4,128,1,float16,float16,0,0.13158399860064188
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,1,128,1,float16,float16,0,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,float16,0,0.07202133536338806
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,4,128,1,float16,float16,0,0.06894933183987935
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,float16,0,0.0769706666469574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,float16,0,0.05017599960168203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,8,2,128,1,float16,float16,0,0.06007466713587443
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,1,128,1,float16,float16,0,0.04420266548792521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,2,128,1,float16,float16,0,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,8,1,128,1,float16,float16,0,0.15940266847610474
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,4,128,1,float16,float16,0,0.04983466863632202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,float16,0,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,8,2,128,1,float16,float16,0,0.09915733337402344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,1,128,1,float16,float16,0,0.72652800877889
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,2,128,1,float16,float16,0,0.9021440347035726
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,float16,0,0.7002453009287516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,4,128,1,float16,float16,0,1.3431466420491536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,1,128,1,float16,float16,0,0.37273601690928143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,8,1,128,1,float16,float16,0,1.4202879269917805
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,float16,0,1.377962589263916
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,8,2,128,1,float16,float16,0,1.7706665992736816
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,2,128,1,float16,float16,0,0.4797439972559611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,4,128,1,float16,float16,0,0.6811306476593018
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,4,128,1,float16,float16,0,0.35123201211293537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,float16,0,0.7048532962799072
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,float16,0,0.3638613224029541
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,float16,0,0.19490132729212442
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,float16,0,1.369258721669515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,float16,0,0.36215468247731525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,2,128,1,float16,float16,0,0.13499733805656433
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,8,4,128,1,float16,float16,0,2.6427733103434243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,float16,0,0.10752000411351521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,2,128,1,float16,float16,0,0.07048533360163371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,float16,0,0.1930239995320638
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,4,128,1,float16,float16,0,0.10171733299891154
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,1,128,1,float16,float16,0,0.10956799983978271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,float16,0,0.1083733340104421
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,1,128,1,float16,float16,0,0.19524266322453818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,float16,0,0.05649066468079885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,1,128,1,float16,float16,0,0.040106666584809623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,2,128,1,float16,float16,0,0.043007999658584595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,8,2,128,1,float16,float16,0,0.24576000372568765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,float16,0,0.05597866574923197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,float16,0,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,8,4,128,1,float16,float16,0,0.05239466826121012
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,1,128,1,float16,float16,0,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,4,128,1,float16,float16,0,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,float16,0,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,1,128,1,float16,float16,0,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,8,2,128,1,float16,float16,0,0.030207999050617218
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,2,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,4,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,8,4,128,1,float16,float16,0,0.18602667252222696
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,1,128,1,float16,float16,0,0.28177066644032794
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,8,1,128,1,float16,float16,0,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,2,128,1,float16,float16,0,0.3887786865234375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,8,1,128,1,float16,float16,0,0.5543253421783447
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,float16,0,0.6161066691080729
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,4,128,1,float16,float16,0,0.5889706611633301
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,1,128,1,float16,float16,0,0.14591999848683676
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,float16,0,0.31334400177001953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,8,2,128,1,float16,float16,0,0.7417173385620117
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,float16,0,0.16776533921559653
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,8,4,128,1,float16,float16,0,1.1687253316243489
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,float16,0,0.31470932563145954
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,4,128,1,float16,float16,0,0.15974400440851846
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,1,128,1,float16,float16,0,0.0820906658967336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,2,128,1,float16,float16,0,0.19848533471425375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,1,128,1,float16,float16,0,0.04437333345413208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,8,4,128,1,float16,float16,0,0.3025919993718465
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,float16,0,0.09284266829490662
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,2,128,1,float16,float16,0,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,float16,0,0.09284266829490662
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,8,4,128,1,float16,float16,0,0.08499200145403545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,2,128,1,float16,float16,0,0.031914666295051575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,2,128,1,float16,float16,0,0.10820266604423523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,1,128,1,float16,float16,0,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,float16,0,0.6104746659596761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,float16,0,0.045226668318112694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,float16,0,0.043178667624791466
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,8,4,128,1,float16,float16,0,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,float16,0,0.028330666323502857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,2,128,1,float16,float16,0,0.02457600086927414
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,float16,0,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,1,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,4,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,2,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,1,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,float16,0,0.16895999511082968
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,float16,0,0.01757866640885671
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,2,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,1,128,1,float16,float16,0,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,4,128,1,float16,float16,0,0.017407999684413273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,float16,0,0.01757866640885671
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,8,4,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,1,128,1,float16,float16,0,0.11844266454378764
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,2,128,1,float16,float16,0,0.1723733345667521
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,4,128,1,float16,float16,0,0.27818665901819867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,float16,0,0.293887992699941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,float16,0,0.15633066495259604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,8,2,128,1,float16,float16,0,0.3548159996668498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,1,128,1,float16,float16,0,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,8,1,128,1,float16,float16,0,0.250709335009257
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,float16,0,0.2959360082944234
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,2,128,1,float16,float16,0,0.09233066439628601
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,1,128,1,float16,float16,0,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,float16,0,0.155648003021876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,8,4,128,1,float16,float16,0,0.5623466571172079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,float16,0,0.0865280032157898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,8,4,128,1,float16,float16,0,0.14728533228238425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,float16,0,0.03942399968703588
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,float16,0,0.08703999718030293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,2,128,1,float16,float16,0,0.04454400142033895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,float16,0,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,4,128,1,float16,float16,0,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,8,4,128,1,float16,float16,0,0.07987200220425923
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,float16,0,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,2,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,1,128,1,float16,float16,0,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,1,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,4,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,2,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,4,128,1,float16,float16,0,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,8,2,128,1,float16,float16,0,0.027818667391935985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,1,128,1,float16,float16,0,0.06843733290831248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,2,128,1,float16,float16,0,0.09352533022562663
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,float16,0,0.15633066495259604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,4,128,1,float16,float16,0,0.1454080045223236
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,float16,0,0.08567466338475545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,1,128,1,float16,float16,0,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,8,1,128,1,float16,float16,0,0.11912533640861511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,8,2,128,1,float16,float16,0,0.17151999473571777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,2,128,1,float16,float16,0,0.04369066655635834
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,8,4,128,1,float16,float16,0,0.27801599105199176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,float16,0,0.155648003021876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,4,128,1,float16,float16,0,0.07645866771539052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,1,128,1,float16,float16,0,0.023893333971500397
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,float16,0,0.040106666584809623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,float16,0,0.08635733524958293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,1,128,1,float16,float16,0,0.017407999684413273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,float16,0,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,2,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,2,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,float16,0,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,4,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,8,4,128,1,float16,float16,0,0.03583999971548716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,2,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,8,4,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,8,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,1,128,1,float16,float16,0,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,8,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,float16,0,0.09847467144330342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,2,128,1,float16,float16,0,0.05853866537412008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,1,128,1,float16,float16,0,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,4,128,1,float16,float16,0,0.09079466263453166
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,float16,0,0.04778666794300079
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,8,1,128,1,float16,float16,0,0.09915733337402344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,1,128,1,float16,float16,0,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,2,128,1,float16,float16,0,0.03293866664171219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,8,4,128,1,float16,float16,0,0.17510400215784708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,float16,0,0.09864532947540283
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,4,128,1,float16,float16,0,0.04147200038035711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,8,2,128,1,float16,float16,0,0.12407466769218445
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,float16,0,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,2,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,1,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,float16,0,0.04795733094215393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,4,128,1,float16,float16,0,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,float16,0,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,1,128,1,float16,float16,0,0.015018666783968607
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,2,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,4,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,float16,0,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,8,4,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,2,128,1,float16,float16,0,0.048298666874567665
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,1,128,1,float16,float16,0,0.04437333345413208
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,float16,0,0.06126933296521505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,4,128,1,float16,float16,0,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,float16,0,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,8,1,128,1,float16,float16,0,0.08004266520341237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,2,128,1,float16,float16,0,0.02935466667016347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,8,2,128,1,float16,float16,0,0.09028266867001851
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,1,128,1,float16,float16,0,0.027647999425729115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,8,4,128,1,float16,float16,0,0.1250986655553182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,float16,0,0.06195199986298879
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,float16,0,0.023381332556406658
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,1,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,2,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,4,128,1,float16,float16,0,0.022869333624839783
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,float16,0,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,8,4,128,1,float16,float16,0,0.03328000009059906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,float16,0,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,1,128,1,float16,float16,0,0.0145066666106383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,2,128,1,float16,float16,0,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,4,128,1,float16,float16,0,0.01672533278663953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,4,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,8,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,0,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,0,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,0,0.03345066557327906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,0,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,0,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,0,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,0,0.051882664362589516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,0,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,0,0.019797333826621372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,0,0.022015998760859173
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,0,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,0,0.02679466704527537
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,float16,0,3.457536061604818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,4,1,128,1,float16,float16,0,3.1185919443766275
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,4,1,128,1,float16,float16,0,6.104746500651042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,float16,0,6.633642832438151
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,4,2,128,1,float16,float16,0,6.369792302449544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,float16,0,6.712319691975911
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,4,2,128,1,float16,float16,0,3.2474454243977866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,float16,0,3.442005475362142
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,float16,0,1.9522560437520344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,4,1,128,1,float16,float16,0,1.7428480784098308
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,4,2,128,1,float16,float16,0,1.8565120697021484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,float16,0,1.9172693888346355
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,4,1,128,1,float16,float16,0,12.2337277730306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,4,2,128,1,float16,float16,0,12.680533091227213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,4,1,128,1,float16,float16,0,3.490645408630371
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,4,2,128,1,float16,float16,0,3.665578524271647
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,float16,0,3.9848960240681968
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,float16,0,2.139647960662842
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,4,1,128,1,float16,float16,0,6.8635304768880205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,4,1,128,1,float16,float16,0,1.853610674540202
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,4,2,128,1,float16,float16,0,1.953279972076416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,float16,0,2.10534397761027
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,4,2,128,1,float16,float16,0,7.066794713338216
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,float16,0,4.139861424763997
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,4,1,128,1,float16,float16,0,1.0410666465759277
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,4,2,128,1,float16,float16,0,1.0832213560740154
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,float16,0,1.1692372957865398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,float16,0,1.1680426597595215
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,float16,0,2.862762769063314
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,float16,0,1.5493119557698567
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,4,1,128,1,float16,float16,0,2.489685376485189
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,4,1,128,1,float16,float16,0,4.7013546625773115
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,4,2,128,1,float16,float16,0,2.59771728515625
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,4,2,128,1,float16,float16,0,5.017258644104004
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,float16,0,2.8441600799560547
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,4,1,128,1,float16,float16,0,1.3530453046162922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,4,2,128,1,float16,float16,0,1.39793062210083
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,4,1,128,1,float16,float16,0,0.740010658899943
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,float16,0,0.8403626283009847
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,4,2,128,1,float16,float16,0,0.7751680215199789
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,float16,0,0.8439466953277588
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,float16,0,1.5281492869059246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,float16,0,1.979904015858968
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,4,1,128,1,float16,float16,0,3.1185919443766275
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,4,2,128,1,float16,float16,0,3.3310718536376953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,float16,0,3.7457920710245767
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,4,1,128,1,float16,float16,0,1.6254293123881023
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,float16,0,3.7024428049723306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,4,2,128,1,float16,float16,0,1.7605973879496257
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,4,1,128,1,float16,float16,0,0.8953173160552979
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,float16,0,1.0733226935068767
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,4,2,128,1,float16,float16,0,0.9456640084584554
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,float16,0,1.0579626560211182
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,4,1,128,1,float16,float16,0,6.325248082478841
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,float16,0,1.9781972567240398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,float16,0,0.5995519955952963
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,4,2,128,1,float16,float16,0,6.648490905761719
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,4,1,128,1,float16,float16,0,0.5160959959030151
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,4,2,128,1,float16,float16,0,0.5502293507258097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,float16,0,0.5947733322779337
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,float16,0,1.2189013163248699
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,4,1,128,1,float16,float16,0,0.9656319618225098
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,4,1,128,1,float16,float16,0,1.828181266784668
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,4,2,128,1,float16,float16,0,2.00601593653361
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,float16,0,2.3038293520609536
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,float16,0,2.310826619466146
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,float16,0,0.6695253054300944
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,4,2,128,1,float16,float16,0,1.0642773310343425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,float16,0,1.195520003636678
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,4,1,128,1,float16,float16,0,3.5505491892496743
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,4,1,128,1,float16,float16,0,0.5625173250834147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,4,2,128,1,float16,float16,0,0.5864106814066569
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,4,2,128,1,float16,float16,0,3.848533312479655
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,float16,0,0.3957759936650594
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,float16,0,0.6710613568623861
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,4,1,128,1,float16,float16,0,0.33826132615407306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,float16,0,0.4068693319956462
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,4,2,128,1,float16,float16,0,0.3587413231531779
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,float16,0,1.2142933209737141
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,4,2,128,1,float16,float16,0,1.881600062052409
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,4,1,128,1,float16,float16,0,1.67031462987264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,4,1,128,1,float16,float16,0,0.8818346659342448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,float16,0,2.3289173444112143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,float16,0,2.3079253832499185
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,4,1,128,1,float16,float16,0,0.482474684715271
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,4,2,128,1,float16,float16,0,0.989525318145752
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,4,2,128,1,float16,float16,0,0.5398186842600504
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,4,2,128,1,float16,float16,0,3.628885269165039
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,float16,0,0.6408533255259196
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,float16,0,0.6360746622085571
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,4,1,128,1,float16,float16,0,3.305983861287435
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,float16,0,1.1922773520151775
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,4,1,128,1,float16,float16,0,0.28040534257888794
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,4,2,128,1,float16,float16,0,0.31692800919214886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,float16,0,0.20667733748753866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,4,1,128,1,float16,float16,0,0.17732266585032144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,float16,0,0.36403199036916095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,float16,0,0.36693334579467773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,float16,0,0.20940800507863364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,4,2,128,1,float16,float16,0,0.1879040002822876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,float16,0,0.7702186902364095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,4,1,128,1,float16,float16,0,0.544426679611206
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,4,2,128,1,float16,float16,0,1.1468799908955891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,float16,0,1.4740479787190754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,4,1,128,1,float16,float16,0,1.016149361928304
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,float16,0,1.4764374097188313
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,4,2,128,1,float16,float16,0,0.6161066691080729
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,4,2,128,1,float16,float16,0,2.2191786766052246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,4,1,128,1,float16,float16,0,0.30498133103052777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,float16,0,0.7772160371144613
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,4,1,128,1,float16,float16,0,1.959935983022054
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,float16,0,0.42291200160980225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,float16,0,0.4164266586303711
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,4,2,128,1,float16,float16,0,0.3461120128631592
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,float16,0,0.2336426575978597
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,float16,0,0.23552000522613525
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,4,2,128,1,float16,float16,0,0.12236799796422322
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,4,2,128,1,float16,float16,0,0.18568533658981323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,4,1,128,1,float16,float16,0,0.1728853384653727
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,4,1,128,1,float16,float16,0,0.1153706709543864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,float16,0,0.13363200426101685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,float16,0,0.13414399822553
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,float16,0,0.8157866795857748
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,4,1,128,1,float16,float16,0,0.9683626492818197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,float16,0,1.58515199025472
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,float16,0,1.5813973744710286
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,4,1,128,1,float16,float16,0,0.504149317741394
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,float16,0,0.42956801255544025
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,4,1,128,1,float16,float16,0,1.8705066045125325
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,4,2,128,1,float16,float16,0,0.6142293214797974
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,4,2,128,1,float16,float16,0,1.156608025232951
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,float16,0,0.8207360108693441
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,4,1,128,1,float16,float16,0,0.27665066719055176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,4,2,128,1,float16,float16,0,0.32819199562072754
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,4,2,128,1,float16,float16,0,2.222933292388916
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,4,2,128,1,float16,float16,0,0.18824533621470133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,float16,0,0.24200532833735147
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,float16,0,0.2392746607462565
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,float16,0,0.1327786644299825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,float16,0,0.0897706647713979
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,float16,0,0.13329066832860312
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,4,2,128,1,float16,float16,0,0.10717866818110149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,4,1,128,1,float16,float16,0,0.15598932902018228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,4,1,128,1,float16,float16,0,0.07714133461316426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,float16,0,0.43110398451487225
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,float16,0,0.0897706647713979
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,4,2,128,1,float16,float16,0,0.08072533210118611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,float16,0,1.0581333637237549
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,4,1,128,1,float16,float16,0,0.09949866930643718
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,float16,0,1.057792027791341
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,4,1,128,1,float16,float16,0,0.6014293432235718
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,4,2,128,1,float16,float16,0,0.7482026418050131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,float16,0,0.5502293507258097
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,4,1,128,1,float16,float16,0,1.162922700246175
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,4,1,128,1,float16,float16,0,0.3186346689860026
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,float16,0,0.5510826508204142
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,4,2,128,1,float16,float16,0,0.4005546569824219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,4,1,128,1,float16,float16,0,0.1776640017827352
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,float16,0,0.16674133141835532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,float16,0,0.29371732473373413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,4,2,128,1,float16,float16,0,1.4049280484517415
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,float16,0,0.07970133423805237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,float16,0,0.1687893271446228
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,4,1,128,1,float16,float16,0,0.10547199845314026
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,4,1,128,1,float16,float16,0,0.06144000093142191
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,4,2,128,1,float16,float16,0,0.12356266379356384
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,4,2,128,1,float16,float16,0,0.2193066676457723
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,float16,0,0.07970133423805237
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,float16,0,0.056661332647005715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,float16,0,0.2935466567675273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,4,1,128,1,float16,float16,0,0.04727466901143392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,4,2,128,1,float16,float16,0,0.05017599960168203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,float16,0,0.057002668579419456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,4,2,128,1,float16,float16,0,0.06758399804433186
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,4,1,128,1,float16,float16,0,0.6085973183314005
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,float16,0,0.6265173355738322
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,4,1,128,1,float16,float16,0,0.3206826647122701
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,4,2,128,1,float16,float16,0,0.7942826747894287
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,float16,0,0.6289066473642985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,4,2,128,1,float16,float16,0,0.4275199969609578
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,float16,0,0.33211733897527057
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,4,1,128,1,float16,float16,0,1.1871573130289714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,float16,0,1.2250453631083171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,4,1,128,1,float16,float16,0,0.17510400215784708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,float16,0,1.2267519632975261
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,float16,0,0.3285333315531413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,float16,0,0.18141865730285645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,4,2,128,1,float16,float16,0,0.2300586700439453
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,float16,0,0.18107734123865762
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,4,2,128,1,float16,float16,0,0.0679253339767456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,4,2,128,1,float16,float16,0,1.5433386166890461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,4,1,128,1,float16,float16,0,0.05836800237496694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,float16,0,0.08618666728337605
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,float16,0,0.05819733440876007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,float16,0,0.05819733440876007
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,4,1,128,1,float16,float16,0,0.045909335215886436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,4,1,128,1,float16,float16,0,0.03498666733503342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,float16,0,0.04027733455101649
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,4,2,128,1,float16,float16,0,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,float16,0,0.040106666584809623
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,4,1,128,1,float16,float16,0,0.09847467144330342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,4,2,128,1,float16,float16,0,0.12731732924779257
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,float16,0,0.08755200107892354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,4,1,128,1,float16,float16,0,0.43724799156188965
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,4,2,128,1,float16,float16,0,0.6357333262761434
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,4,2,128,1,float16,float16,0,0.04966400067011515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,float16,0,0.5376000006993612
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,4,1,128,1,float16,float16,0,0.22886399428049722
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,float16,0,1.0525013605753581
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,4,2,128,1,float16,float16,0,0.3331413269042969
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,4,1,128,1,float16,float16,0,0.12492799758911133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,float16,0,0.28074665864308673
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,4,1,128,1,float16,float16,0,0.8444586594899496
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,float16,0,0.5379413366317749
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,float16,0,1.0482347011566162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,float16,0,0.15240533153216043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,float16,0,0.2797226707140605
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,float16,0,0.07441066702206929
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,4,2,128,1,float16,float16,0,1.2086613178253174
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,4,1,128,1,float16,float16,0,0.04232533276081085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,4,2,128,1,float16,float16,0,0.051029334465662636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,4,1,128,1,float16,float16,0,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,4,2,128,1,float16,float16,0,0.17851734161376953
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,float16,0,0.04232533276081085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,4,1,128,1,float16,float16,0,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,4,2,128,1,float16,float16,0,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,float16,0,0.02867199977238973
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,4,1,128,1,float16,float16,0,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,4,2,128,1,float16,float16,0,0.02491733431816101
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,float16,0,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,4,2,128,1,float16,float16,0,0.09796266754468282
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,float16,0,0.0288426677385966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,float16,0,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,float16,0,0.15103999773661295
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,4,1,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,4,2,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,float16,0,0.024746666351954143
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,float16,0,0.07167999943097432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,4,1,128,1,float16,float16,0,0.18193066120147705
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,4,2,128,1,float16,float16,0,0.28706133365631104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,float16,0,0.4872533480326335
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,float16,0,0.25446399052937824
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,float16,0,0.49032533168792725
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,4,1,128,1,float16,float16,0,0.09745066364606221
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,4,2,128,1,float16,float16,0,0.15308800339698792
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,float16,0,0.13619200388590494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,4,1,128,1,float16,float16,0,0.34935466448465985
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,4,2,128,1,float16,float16,0,0.5434026718139648
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,4,2,128,1,float16,float16,0,0.08277333279450734
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,float16,0,0.25309866666793823
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,float16,0,0.1353386640548706
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,4,1,128,1,float16,float16,0,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,4,1,128,1,float16,float16,0,0.05256533126036326
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,float16,0,0.06434133152167003
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,float16,0,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,float16,0,0.023210667073726654
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,float16,0,0.03601066768169403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,4,2,128,1,float16,float16,0,0.027477333943049114
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,4,1,128,1,float16,float16,0,0.017407999684413273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,4,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,4,2,128,1,float16,float16,0,0.03942399968703588
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,4,1,128,1,float16,float16,0,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,float16,0,0.023039999107519787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,4,1,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,4,2,128,1,float16,float16,0,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,float16,0,0.017407999684413273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,4,1,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,4,2,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,float16,0,0.03618133316437403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,float16,0,0.017407999684413273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,4,1,128,1,float16,float16,0,0.04488533238569895
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,float16,0,0.13056000073750815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,4,1,128,1,float16,float16,0,0.08328533172607422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,4,2,128,1,float16,float16,0,0.13909332950909933
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,float16,0,0.24473599592844644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,4,2,128,1,float16,float16,0,0.07287466526031494
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,float16,0,0.24337067206700644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,float16,0,0.13038933277130127
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,float16,0,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,4,1,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,float16,0,0.05597866574923197
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,4,2,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,4,2,128,1,float16,float16,0,0.03532800078392029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,float16,0,0.031231999397277832
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,4,2,128,1,float16,float16,0,0.26658133665720624
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,4,1,128,1,float16,float16,0,0.16366933782895407
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,4,1,128,1,float16,float16,0,0.019285333653291065
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,float16,0,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,4,1,128,1,float16,float16,0,0.014677333335081736
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,4,2,128,1,float16,float16,0,0.01621333385507266
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,4,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,4,2,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,4,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,4,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,4,2,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,4,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,4,1,128,1,float16,float16,0,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,float16,0,0.05358933409055074
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,4,1,128,1,float16,float16,0,0.04249600072701772
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,4,2,128,1,float16,float16,0,0.07389866809050243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,float16,0,0.12782933314641318
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,float16,0,0.1293653349081675
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,4,2,128,1,float16,float16,0,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,float16,0,0.05376000205675761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,float16,0,0.030720000465710957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,4,1,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,4,1,128,1,float16,float16,0,0.08447999755541484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,float16,0,0.031061333914597828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,4,2,128,1,float16,float16,0,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,4,2,128,1,float16,float16,0,0.1358506679534912
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,4,1,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,float16,0,0.019626667102177937
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,4,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,4,2,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,4,2,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,4,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,4,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,4,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,4,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,4,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,4,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,float16,0,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,float16,0,0.0602453351020813
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,4,1,128,1,float16,float16,0,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,4,2,128,1,float16,float16,0,0.040789333482583366
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,float16,0,0.06058666606744131
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,4,1,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,float16,0,0.021674667795499165
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,4,1,128,1,float16,float16,0,0.016554666062196095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,4,2,128,1,float16,float16,0,0.025941332181294758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,float16,0,0.034815999368826546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,4,2,128,1,float16,float16,0,0.0885759989420573
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,4,1,128,1,float16,float16,0,0.056661332647005715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,4,2,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,4,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,4,2,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,4,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,4,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,float16,0,0.014335999886194864
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,4,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,4,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,4,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,4,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,4,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,4,2,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,float16,0,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,float16,0,0.04130133241415024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,4,1,128,1,float16,float16,0,0.02918400118748347
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,float16,0,0.04130133241415024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,4,2,128,1,float16,float16,0,0.03259733319282532
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,4,1,128,1,float16,float16,0,0.02065066620707512
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,float16,0,0.0264533335963885
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,4,1,128,1,float16,float16,0,0.014848000059525171
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,4,2,128,1,float16,float16,0,0.016384000579516094
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,4,2,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,4,1,128,1,float16,float16,0,0.04727466901143392
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,4,2,128,1,float16,float16,0,0.05580799778302511
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,4,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,4,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,4,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,4,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,4,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,4,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,4,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,4,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,4,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,4,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,0,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,0,0.019968000551064808
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,0,0.012970666090647379
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,0,0.0339626669883728
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,0,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,2,1,128,1,float16,float16,0,1.7520640691121419
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,float16,0,1.8609493573506672
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,float16,0,1.8631679217020671
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,2,1,128,1,float16,float16,0,3.1766185760498047
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,float16,0,3.3880745569864907
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,float16,0,3.3914880752563477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,float16,0,1.0658133029937744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,2,1,128,1,float16,float16,0,1.014954646428426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,float16,0,1.070250670115153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,float16,0,1.117695967356364
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,2,1,128,1,float16,float16,0,1.0576213200887044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16384,2,1,128,1,float16,float16,0,6.2192637125651045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,2,1,128,1,float16,float16,0,1.9053227106730144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,float16,0,2.04475736618042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,float16,0,2.032639980316162
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,float16,0,0.7082666556040446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,float16,0,1.1356159845987956
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,2,1,128,1,float16,float16,0,0.6920533180236816
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,float16,0,0.711850643157959
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,12288,2,1,128,1,float16,float16,0,3.576149304707845
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,float16,0,1.4791679382324219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,2,1,128,1,float16,float16,0,0.7666347026824951
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,float16,0,0.8656213283538818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,2,1,128,1,float16,float16,0,1.3822293281555176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,float16,0,0.8314879735310873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,float16,0,1.480021317799886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,10240,2,1,128,1,float16,float16,0,2.5518080393473306
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,float16,0,0.529578685760498
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,2,1,128,1,float16,float16,0,0.494762659072876
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,float16,0,0.5348693529764811
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,float16,0,1.0246826807657878
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,float16,0,1.8831359545389812
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,2,1,128,1,float16,float16,0,1.6926719347635906
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,2,1,128,1,float16,float16,0,0.9245013395945231
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,float16,0,1.0489173730214436
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,float16,0,1.896959940592448
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,float16,0,0.5855573415756226
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,2,1,128,1,float16,float16,0,0.5379413366317749
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,float16,0,0.37905065218607586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,2,1,128,1,float16,float16,0,0.35891199111938477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,float16,0,0.37563733259836835
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,float16,0,0.5876053174336752
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,8192,2,1,128,1,float16,float16,0,3.227135976155599
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,float16,0,0.6570666631062826
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,float16,0,0.3925333420435588
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,2,1,128,1,float16,float16,0,0.5809493462244669
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,float16,0,1.180842638015747
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,float16,0,0.6464853286743164
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,float16,0,1.174015998840332
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,6144,2,1,128,1,float16,float16,0,1.9140267372131348
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,2,1,128,1,float16,float16,0,0.34986666838328045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,float16,0,0.24217599630355835
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,2,1,128,1,float16,float16,0,1.025706688563029
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,2,1,128,1,float16,float16,0,0.22784000635147095
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,float16,0,0.23995733261108398
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,float16,0,0.3916800022125244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,float16,0,0.3566933472951253
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,float16,0,0.6198613246281942
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,2,1,128,1,float16,float16,0,0.5179733435312907
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,float16,0,0.6186666488647461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,float16,0,1.1311786969502766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,float16,0,1.1352746486663818
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,2,1,128,1,float16,float16,0,0.9588053226470947
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,2,1,128,1,float16,float16,0,0.3037866751352946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,float16,0,0.3573760191599528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,float16,0,0.202239990234375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,float16,0,0.20667733748753866
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,float16,0,0.1367039978504181
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,2,1,128,1,float16,float16,0,0.1812480092048645
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,4096,2,1,128,1,float16,float16,0,1.8085546493530273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,float16,0,0.1353386640548706
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,2,1,128,1,float16,float16,0,0.12714667121569315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,float16,0,0.731818675994873
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,float16,0,0.7219200134277344
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,float16,0,0.40908801555633545
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,float16,0,0.40516265233357746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,2,1,128,1,float16,float16,0,0.3300693432490031
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,2,1,128,1,float16,float16,0,0.5860693454742432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,float16,0,0.22920533021291098
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,2,1,128,1,float16,float16,0,0.18244266510009766
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,float16,0,0.1327786644299825
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,float16,0,0.23142399390538534
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,2,1,128,1,float16,float16,0,0.12014933427174886
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,float16,0,0.09045333663622539
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,2,1,128,1,float16,float16,0,0.08447999755541484
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,float16,0,0.09079466263453166
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,3072,2,1,128,1,float16,float16,0,1.0953386624654133
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,float16,0,0.13209600249926248
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,float16,0,0.4130133390426636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,2,1,128,1,float16,float16,0,0.3099306623140971
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,float16,0,0.4128426710764567
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,2,1,128,1,float16,float16,0,0.5679786602656046
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,float16,0,0.23347200949986777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,2,1,128,1,float16,float16,0,0.1776640017827352
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,float16,0,0.7522986729939779
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,2,1,128,1,float16,float16,0,0.10820266604423523
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,float16,0,0.12987732887268066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,float16,0,0.23193599780400595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,float16,0,0.7596373558044434
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,float16,0,0.12868266304334006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,2,1,128,1,float16,float16,0,0.08072533210118611
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,float16,0,0.08925867080688477
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,float16,0,0.06297599772612254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,float16,0,0.0628053347269694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,2048,2,1,128,1,float16,float16,0,1.0859519640604656
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,float16,0,0.08942932883898418
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,2,1,128,1,float16,float16,0,0.05905066430568695
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,2,1,128,1,float16,float16,0,0.36693334579467773
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,2,1,128,1,float16,float16,0,0.202239990234375
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,float16,0,0.28177066644032794
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,float16,0,0.1609386702378591
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,float16,0,0.5099519888559977
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,float16,0,0.5145599842071533
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,float16,0,0.28177066644032794
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,2,1,128,1,float16,float16,0,0.11690666278203328
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1536,2,1,128,1,float16,float16,0,0.6894933382670084
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,float16,0,0.16247466206550598
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,float16,0,0.07867733140786488
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,2,1,128,1,float16,float16,0,0.05017599960168203
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,2,1,128,1,float16,float16,0,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,float16,0,0.049322664737701416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,float16,0,0.05614933371543884
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,float16,0,0.04915200173854828
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,float16,0,0.07816533247629802
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,2,1,128,1,float16,float16,0,0.06553600231806438
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,float16,0,0.056320001681645714
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,2,1,128,1,float16,float16,0,0.20736000935236612
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,2,1,128,1,float16,float16,0,0.3839999834696452
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,float16,0,0.31197865804036456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,float16,0,0.31163734197616577
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,float16,0,0.17425066232681274
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,2,1,128,1,float16,float16,0,0.11758933464686076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,float16,0,0.5746346712112427
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,float16,0,0.5797546704610189
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,float16,0,0.08362666765848796
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,float16,0,0.08584533135096233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,float16,0,0.17220266660054526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,float16,0,0.05734399954477946
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,2,1,128,1,float16,float16,0,0.06604800124963124
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1024,2,1,128,1,float16,float16,0,0.7321600119272867
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,float16,0,0.039594667653242745
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,float16,0,0.03601066768169403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,2,1,128,1,float16,float16,0,0.0363520011305809
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,2,1,128,1,float16,float16,0,0.034304000437259674
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,float16,0,0.03976533313592275
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,2,1,128,1,float16,float16,0,0.04863999783992767
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,float16,0,0.056832000613212585
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,float16,0,0.2640213370323181
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,2,1,128,1,float16,float16,0,0.15820800264676413
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,float16,0,0.03601066768169403
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,2,1,128,1,float16,float16,0,0.2916693290074666
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,float16,0,0.2616320053736369
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,float16,0,0.14353066682815552
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,2,1,128,1,float16,float16,0,0.08755200107892354
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,float16,0,0.47257598241170246
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,float16,0,0.1462613344192505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,float16,0,0.47274665037790936
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,float16,0,0.067071999112765
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,float16,0,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,2,1,128,1,float16,float16,0,0.0481279989083608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,float16,0,0.07167999943097432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,float16,0,0.04113066693147024
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,float16,0,0.027989332874615986
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,2,1,128,1,float16,float16,0,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,float16,0,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,2,1,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,float16,0,0.024405332903067272
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,float16,0,0.022698665658632915
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,2,1,128,1,float16,float16,0,0.021162666380405426
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,2,1,128,1,float16,float16,0,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,512,2,1,128,1,float16,float16,0,0.5570559899012247
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,float16,0,0.028160000840822857
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,float16,0,0.12544000148773193
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,float16,0,0.1288533310095469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,2,1,128,1,float16,float16,0,0.07133866846561432
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,2,1,128,1,float16,float16,0,0.13192533453305563
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,float16,0,0.24012800057729086
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,float16,0,0.056661332647005715
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,float16,0,0.23586134115854898
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,float16,0,0.05478399991989136
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,2,1,128,1,float16,float16,0,0.0365226666132609
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,float16,0,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,float16,0,0.03515733281771342
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,2,1,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,2,1,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,float16,0,0.018944000204404194
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,2,1,128,1,float16,float16,0,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,256,2,1,128,1,float16,float16,0,0.24661332368850708
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,float16,0,0.0170666662355264
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,2,1,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,float16,0,0.015360000232855478
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,2,1,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,float16,0,0.017237332959969837
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,float16,0,0.051370665431022644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,2,1,128,1,float16,float16,0,0.03276800115903219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,float16,0,0.05034666756788889
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,float16,0,0.12288000186284383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,float16,0,0.030037333567937214
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,2,1,128,1,float16,float16,0,0.06382933259010315
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,2,1,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,float16,0,0.12288000186284383
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,float16,0,0.030378667016824085
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,2,1,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,float16,0,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,float16,0,0.0194560003777345
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,2,1,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,2,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,2,1,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,128,2,1,128,1,float16,float16,0,0.12219732999801636
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,2,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,2,1,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,float16,0,0.029696000119050343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,float16,0,0.029696000119050343
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,float16,0,0.049322664737701416
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,2,1,128,1,float16,float16,0,0.032085334261258446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,2,1,128,1,float16,float16,0,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,float16,0,0.051370665431022644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,2,1,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,float16,0,0.013994666437307993
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,2,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,2,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,2,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,64,2,1,128,1,float16,float16,0,0.06451199948787689
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,2,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,float16,0,0.020992000897725422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,2,1,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,float16,0,0.021333334346612293
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,2,1,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,float16,0,0.03362133353948593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,2,1,128,1,float16,float16,0,0.025258667767047882
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,float16,0,0.03379199902216593
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,float16,0,0.01570133368174235
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,2,1,128,1,float16,float16,0,0.012458667159080505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,2,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,2,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,2,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,2,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,32,2,1,128,1,float16,float16,0,0.037717332442601524
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,2,1,128,1,float16,float16,0,0.016042667130629223
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,float16,0,0.01791999985774358
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,float16,0,0.018090666582187016
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,2,1,128,1,float16,float16,0,0.02218666672706604
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,float16,0,0.025770666698614757
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,2,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,2,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,2,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,2,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,16,2,1,128,1,float16,float16,0,0.031744000812371574
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,2,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,2,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,0,0.01331199953953425
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,0,0.015872000406185787
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,0,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,0,0.012800000607967377
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,0,0.020138667275508244
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,float16,0,0.6903466383616129
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,float16,0,1.0439679622650146
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,float16,0,1.0453333059946697
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,float16,0,0.6929066975911459
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,float16,0,1.1125760078430176
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,float16,0,1.8129919370015461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,float16,0,1.8146986961364746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,float16,0,0.3742719888687134
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,float16,0,0.6891520023345947
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,float16,0,0.6906879742940267
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,float16,0,1.0912426312764485
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,float16,0,0.37597866853078205
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,float16,0,0.5114880005518595
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,float16,0,0.8017919858296713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,float16,0,0.8203946749369303
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,float16,0,0.2998613317807515
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,float16,0,0.516266663869222
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,float16,0,0.30037333567937213
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,float16,0,0.3717120091120402
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,float16,0,0.9977173010508219
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,float16,0,0.22579199075698853
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,float16,0,0.3679573138554891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,float16,0,0.564736008644104
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,float16,0,0.5739519993464152
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,float16,0,0.9927679697672526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,float16,0,0.2254506746927897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,float16,0,0.3742719888687134
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,float16,0,0.23654399315516153
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,float16,0,0.3722240130106608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,float16,0,0.6096213261286417
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,float16,0,0.620031992594401
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,float16,0,0.159061332543691
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,float16,0,0.23603200912475586
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,float16,0,0.15923200050989786
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,float16,0,0.13260799646377563
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,float16,0,0.19882667064666748
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,float16,0,0.1996799906094869
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,float16,0,0.13312000036239624
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,float16,0,0.33484800656636554
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,float16,0,0.10854400197664897
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,float16,0,0.33023999134699505
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,float16,0,0.10871466994285583
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,float16,0,0.5758293469746908
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,float16,0,0.5807786782582601
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,float16,0,0.21248000860214233
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,float16,0,0.12851199507713318
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,float16,0,0.2177706758181254
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,float16,0,0.12868266304334006
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,float16,0,0.3752959966659546
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,float16,0,0.3722240130106608
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,float16,0,0.08345599969228108
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,float16,0,0.08840533097585042
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,float16,0,0.08823466300964355
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,float16,0,0.08328533172607422
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,float16,0,0.08601599931716919
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,float16,0,0.12526933352152506
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,float16,0,0.12390399972597758
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,float16,0,0.21094399690628052
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,float16,0,0.37495466073354083
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,float16,0,0.21452800432840982
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,float16,0,0.3742719888687134
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,float16,0,0.08601599931716919
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,float16,0,0.06195199986298879
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,float16,0,0.06195199986298879
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,float16,0,0.05870933334032694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,float16,0,0.05870933334032694
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,float16,0,0.07321600119272868
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,float16,0,0.14574933052062988
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,float16,0,0.07372800012429555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,float16,0,0.1609386702378591
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,float16,0,0.26333866516749066
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,float16,0,0.2524159948031108
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,float16,0,0.05376000205675761
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,float16,0,0.053930665055910744
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,float16,0,0.04863999783992767
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,float16,0,0.048810665806134544
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,float16,0,0.04607999821503957
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,float16,0,0.04625066618124644
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,float16,0,0.08191999793052673
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,float16,0,0.0820906658967336
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,float16,0,0.15001599987347922
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,float16,0,0.1518933375676473
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,float16,0,0.054101333022117615
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,float16,0,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,float16,0,0.2691413362820943
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,float16,0,0.03908266623814901
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,float16,0,0.038912000755469
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,float16,0,0.03566933423280716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,float16,0,0.27033599217732746
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,float16,0,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,float16,0,0.0341333324710528
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,float16,0,0.03583999971548716
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,float16,0,0.03822933385769526
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,float16,0,0.0653653343518575
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,float16,0,0.06519466638565063
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,float16,0,0.12134400010108948
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,float16,0,0.12185600399971008
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,float16,0,0.027136000494162243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,float16,0,0.03839999934037527
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,float16,0,0.21998933951059976
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,float16,0,0.022357332209746044
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,float16,0,0.024234667420387268
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,float16,0,0.21913599967956543
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,float16,0,0.0240639994541804
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,float16,0,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,float16,0,0.02252800017595291
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,float16,0,0.020821332931518555
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,float16,0,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,float16,0,0.032255999743938446
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,float16,0,0.054272000988324486
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,float16,0,0.021503999829292297
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,float16,0,0.052906667192777
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,float16,0,0.02184533327817917
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,float16,0,0.10700800021489461
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,float16,0,0.10717866818110149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,float16,0,0.016895999511082966
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,float16,0,0.018602666755517323
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,float16,0,0.01877333347996076
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,float16,0,0.027306665976842243
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,float16,0,0.015530666957298914
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,float16,0,0.05000533163547516
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,float16,0,0.04795733094215393
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,float16,0,0.018432000031073887
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,float16,0,0.01826133330663045
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,float16,0,0.0266239990790685
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,float16,0,0.026965332527955372
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,float16,0,0.013482666263977686
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,float16,0,0.013141332815090815
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,float16,0,0.015189333508412043
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,float16,0,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,float16,0,0.011605333536863327
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,float16,0,0.020479999482631683
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,float16,0,0.011946666985750198
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,float16,0,0.012117333710193634
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,float16,0,0.01228800043463707
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,float16,0,0.013653332988421122
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,float16,0,0.017407999684413273
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,float16,0,0.017749333133300144
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,0,0.011434666812419891
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,0,0.013823999712864557
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,0,0.010751999914646149
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,0,0.010922666639089584
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,0,0.010581333190202713
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,0,0.01109333336353302
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,0,0.014165333161751429
VLLM,0.12.0,NVIDIA A100-SXM4-80GB,context_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,0,0.010751999914646149
