framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,1,128,1,float16,float16,0,60.649454752604164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,1,128,1,float16,fp8,0,54.277069091796875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,4,128,1,float16,fp8,0,55.26176452636719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,4,128,1,float16,float16,0,60.35311381022135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,2,128,1,float16,float16,0,61.36424763997396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,2,128,1,float16,fp8,0,76.09148661295573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,8,128,1,float16,float16,0,59.3126220703125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,64,128,1,float16,fp8,0,31.87224578857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,1,128,1,float16,fp8,0,29.554458618164062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,64,8,128,1,float16,fp8,0,55.4637451171875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,1,128,1,float16,float16,0,35.834757486979164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,64,128,1,float16,float16,0,34.68083699544271
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,2,128,1,float16,float16,0,33.55242156982422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,4,128,1,float16,float16,0,34.26058705647787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,64,128,1,float16,fp8,0,14.488564809163412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,2,128,1,float16,fp8,0,41.098645528157554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,64,128,1,float16,float16,0,16.026976267496746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,1,128,1,float16,float16,0,15.464986165364584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,4,128,1,float16,fp8,0,27.681060791015625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,8,128,1,float16,float16,0,29.267295837402344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,1,128,1,float16,fp8,0,13.667669932047525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,64,8,128,1,float16,fp8,0,30.512532552083332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,2,128,1,float16,float16,0,15.366527557373047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,4,128,1,float16,fp8,0,13.907829284667969
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,4,128,1,float16,float16,0,15.493754069010416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,2,128,1,float16,fp8,0,19.70479965209961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,64,128,1,float16,float16,0,8.500207901000977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,64,128,1,float16,fp8,0,7.3039093017578125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,8,128,1,float16,float16,0,15.983924865722656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,1,128,1,float16,float16,0,7.811386744181315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,1,128,1,float16,fp8,0,6.876288096110026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,64,8,128,1,float16,fp8,0,14.004730224609375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,2,128,1,float16,float16,0,7.735087712605794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,4,128,1,float16,fp8,0,6.948122660319011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,4,128,1,float16,float16,0,7.431770960489909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,2,128,1,float16,fp8,0,9.871552149454752
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,8,128,1,float16,float16,0,7.251637140909831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,64,8,128,1,float16,fp8,0,7.067818959554036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,1,128,1,float16,fp8,0,32.53018697102865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,1,128,1,float16,float16,0,36.723609924316406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,2,128,1,float16,fp8,0,31.212112426757812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,2,128,1,float16,float16,0,37.44446309407552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,8,128,1,float16,fp8,0,32.39644877115885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,4,128,1,float16,float16,0,35.34691619873047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,8,128,1,float16,float16,0,36.1520741780599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,64,128,1,float16,float16,0,19.980127970377605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,64,4,128,1,float16,fp8,0,48.283976236979164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,1,128,1,float16,fp8,0,15.79092788696289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,64,128,1,float16,fp8,0,16.833765665690105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,1,128,1,float16,float16,0,18.139205932617188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,2,128,1,float16,fp8,0,16.181973775227863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,2,128,1,float16,float16,0,18.930789947509766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,4,128,1,float16,float16,0,17.833338419596355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,64,128,1,float16,fp8,0,8.508960088094076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,64,128,1,float16,float16,0,9.226277033487955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,1,128,1,float16,float16,0,9.352287928263346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,1,128,1,float16,fp8,0,7.905722935994466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,4,128,1,float16,fp8,0,15.93875249226888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,2,128,1,float16,float16,0,8.667407989501953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,8,128,1,float16,fp8,0,16.136394500732422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,2,128,1,float16,fp8,0,7.9201920827229815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,4,128,1,float16,float16,0,8.937082926432291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,64,128,1,float16,float16,0,4.4699093500773115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,64,128,1,float16,fp8,0,4.247407913208008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,4,128,1,float16,fp8,0,8.534645080566406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,8,128,1,float16,float16,0,8.951471964518229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,1,128,1,float16,float16,0,4.429989178975423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,64,8,128,1,float16,fp8,0,8.003503799438477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,1,128,1,float16,fp8,0,4.340730667114258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,4,128,1,float16,float16,0,4.092879931131999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,2,128,1,float16,float16,0,4.068714777628581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,2,128,1,float16,fp8,0,3.9875787099202475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,4,128,1,float16,fp8,0,4.213882764180501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,64,8,128,1,float16,float16,0,32.63109334309896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,8,128,1,float16,float16,0,4.287194569905599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,64,8,128,1,float16,fp8,0,4.059610684712728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,1,128,1,float16,fp8,0,22.032480875651043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,1,128,1,float16,float16,0,24.524869283040363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,2,128,1,float16,fp8,0,22.043978373209637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,8,128,1,float16,fp8,0,22.475077311197918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,2,128,1,float16,float16,0,24.548634847005207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,4,128,1,float16,float16,0,24.747360229492188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,8,128,1,float16,float16,0,23.942138671875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,64,128,1,float16,float16,0,13.765584309895834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,64,128,1,float16,fp8,0,12.040203094482422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,1,128,1,float16,float16,0,12.85757319132487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,1,128,1,float16,fp8,0,10.999445597330729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,2,128,1,float16,float16,0,12.049733479817709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,2,128,1,float16,fp8,0,11.052815755208334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,4,128,1,float16,float16,0,12.780682881673178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,64,4,128,1,float16,fp8,0,36.75400034586588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,64,128,1,float16,float16,0,6.511781056722005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,1,128,1,float16,float16,0,6.17409070332845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,64,128,1,float16,fp8,0,6.520997365315755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,1,128,1,float16,fp8,0,5.8426558176676435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,4,128,1,float16,fp8,0,11.302805582682291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,8,128,1,float16,fp8,0,11.344629923502604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,64,8,128,1,float16,float16,0,12.556512196858725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,2,128,1,float16,fp8,0,5.8393599192301435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,4,128,1,float16,float16,0,6.233813603719075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,4,128,1,float16,fp8,0,5.694298426310222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,64,128,1,float16,float16,0,3.096720059712728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,64,128,1,float16,fp8,0,3.329013188680013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,8,128,1,float16,float16,0,5.873578389485677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,1,128,1,float16,float16,0,2.8235254287719727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,8,128,1,float16,fp8,0,5.707578659057617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,64,2,128,1,float16,float16,0,12.963082631429037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,1,128,1,float16,fp8,0,2.798757235209147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,2,128,1,float16,float16,0,3.0149014790852866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,2,128,1,float16,fp8,0,2.897157351175944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,4,128,1,float16,float16,0,2.980442682902018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,8,128,1,float16,float16,0,2.8200906117757163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,4,128,1,float16,fp8,0,2.9604533513387046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,64,8,128,1,float16,fp8,0,4.1589813232421875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,1,128,1,float16,fp8,0,29.588719685872395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,1,128,1,float16,float16,0,31.281514485677082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,2,128,1,float16,fp8,0,28.70300801595052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,2,128,1,float16,float16,0,31.87494405110677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,4,128,1,float16,fp8,0,29.42759958902995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,4,128,1,float16,float16,0,31.934710184733074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,8,128,1,float16,fp8,0,29.50701395670573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,64,128,1,float16,float16,0,18.25930658976237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,1,128,1,float16,fp8,0,14.344341278076172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,1,128,1,float16,float16,0,16.37063980102539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,64,128,1,float16,fp8,0,15.935674031575521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,2,128,1,float16,float16,0,15.191354115804037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,2,128,1,float16,fp8,0,14.419077555338541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,64,128,1,float16,fp8,0,8.31486956278483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,64,128,1,float16,float16,0,9.383125305175781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,1,128,1,float16,float16,0,7.691392262776692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,4,128,1,float16,float16,0,16.628368377685547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,8,128,1,float16,fp8,0,14.766309102376303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,4,128,1,float16,fp8,0,14.81927490234375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,64,8,128,1,float16,float16,0,17.03830337524414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,1,128,1,float16,fp8,0,7.193050384521484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,2,128,1,float16,float16,0,7.385503768920898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,2,128,1,float16,fp8,0,7.353189468383789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,4,128,1,float16,float16,0,8.011765162150065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,64,128,1,float16,float16,0,4.046117464701335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,4,128,1,float16,fp8,0,7.413418451944987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,1,128,1,float16,float16,0,3.7763894399007163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,8,128,1,float16,float16,0,8.214314778645834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,1,128,1,float16,fp8,0,4.052778561909993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,64,128,1,float16,fp8,0,4.980640093485515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,64,8,128,1,float16,fp8,0,7.872586568196614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,64,8,128,1,float16,float16,0,83.41072591145833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,2,128,1,float16,fp8,0,3.643482526143392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,4,128,1,float16,float16,0,3.956512133280436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,2,128,1,float16,float16,0,3.7695093154907227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,4,128,1,float16,fp8,0,4.011952082316081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,64,128,1,float16,float16,0,2.0741920471191406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,8,128,1,float16,float16,0,3.733386675516764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,64,8,128,1,float16,fp8,0,3.773765246073405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,64,128,1,float16,fp8,0,2.3215840657552085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,2,128,1,float16,float16,0,1.9680372873942058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,1,128,1,float16,float16,0,1.8430719375610352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,1,128,1,float16,fp8,0,2.0560426712036133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,2,128,1,float16,fp8,0,1.9065759976704915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,4,128,1,float16,float16,0,1.9124959309895833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,4,128,1,float16,fp8,0,2.061274687449137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,8,128,1,float16,float16,0,1.8518560727437336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,64,8,128,1,float16,fp8,0,2.5061119397481284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,1,128,1,float16,fp8,0,16.63378651936849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,1,128,1,float16,float16,0,18.234036763509113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,2,128,1,float16,fp8,0,16.761056264241535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,2,128,1,float16,float16,0,18.36072540283203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,4,128,1,float16,fp8,0,17.312357584635418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,8,128,1,float16,float16,0,18.220799763997395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,4,128,1,float16,float16,0,19.270464579264324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,1,128,1,float16,float16,0,9.807408014933268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,1,128,1,float16,fp8,0,8.356000264485678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,64,128,1,float16,float16,0,10.514394760131836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,64,128,1,float16,fp8,0,9.564992268880209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,2,128,1,float16,float16,0,9.804746627807617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,2,128,1,float16,fp8,0,8.411712010701498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,64,8,128,1,float16,fp8,0,28.126820882161457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,4,128,1,float16,float16,0,9.41542943318685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,1,128,1,float16,float16,0,4.19264539082845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,64,128,1,float16,float16,0,4.852202733357747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,64,128,1,float16,fp8,0,4.8244320551554365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,2,128,1,float16,float16,0,4.4591414133707685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,8,128,1,float16,float16,0,8.849749247233072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,4,128,1,float16,fp8,0,8.717013041178385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,1,128,1,float16,fp8,0,6.753882726033528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,64,8,128,1,float16,fp8,0,8.77836799621582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,2,128,1,float16,fp8,0,4.608847935994466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,4,128,1,float16,float16,0,4.537706693013509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,4,128,1,float16,fp8,0,4.3865814208984375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,64,128,1,float16,float16,0,2.428757349650065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,8,128,1,float16,float16,0,4.422975858052571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,1,128,1,float16,float16,0,2.114506721496582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,64,128,1,float16,fp8,0,3.3019892374674478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,1,128,1,float16,fp8,0,2.2817440032958984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,64,8,128,1,float16,fp8,0,4.964864095052083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,2,128,1,float16,float16,0,2.1373866399129233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,2,128,1,float16,fp8,0,2.1480372746785483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,4,128,1,float16,float16,0,2.19815460840861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,4,128,1,float16,fp8,0,2.3795092900594077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,8,128,1,float16,float16,0,2.1504480044047036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,64,128,1,float16,float16,0,1.2384693622589111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,64,128,1,float16,fp8,0,1.2645706335703533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,1,128,1,float16,float16,0,1.069381316502889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,64,8,128,1,float16,fp8,0,2.854074796040853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,1,128,1,float16,fp8,0,1.094607988993327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,2,128,1,float16,float16,0,1.0934133529663086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,2,128,1,float16,fp8,0,1.1540053685506184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,4,128,1,float16,float16,0,1.2144052982330322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,4,128,1,float16,fp8,0,1.1422879695892334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,8,128,1,float16,float16,0,1.0929120381673176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,64,8,128,1,float16,fp8,0,1.324629306793213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,1,128,1,float16,float16,0,17.44880549112956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,2,128,1,float16,float16,0,17.92861811319987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,2,128,1,float16,fp8,0,15.879130045572916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,1,128,1,float16,fp8,0,15.728394826253256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,4,128,1,float16,float16,0,17.73536554972331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,4,128,1,float16,fp8,0,16.755813598632812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,1,128,1,float16,float16,0,8.252682367960611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,1,128,1,float16,fp8,0,7.903461456298828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,64,128,1,float16,float16,0,9.777429580688477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,2,128,1,float16,float16,0,8.457349141438803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,64,128,1,float16,fp8,0,9.48732821146647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,2,128,1,float16,fp8,0,7.916234970092773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,8,128,1,float16,fp8,0,16.692095438639324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,64,128,1,float16,float16,0,4.962149302164714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,64,128,1,float16,fp8,0,4.720250765482585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,4,128,1,float16,float16,0,8.253087997436523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,4,128,1,float16,fp8,0,8.36298688252767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,8,128,1,float16,float16,0,8.681157430013021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,64,8,128,1,float16,fp8,0,8.442469278971354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,64,8,128,1,float16,float16,0,39.87230428059896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,1,128,1,float16,float16,0,3.911738713582357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,2,128,1,float16,float16,0,4.028842608133952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,1,128,1,float16,fp8,0,4.279136021931966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,4,128,1,float16,float16,0,4.318367958068848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,2,128,1,float16,fp8,0,4.601962725321452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,8,128,1,float16,float16,0,4.0252641042073565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,64,128,1,float16,float16,0,2.53658660252889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,4,128,1,float16,fp8,0,4.693600018819173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,64,128,1,float16,fp8,0,2.5609119733174643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,1,128,1,float16,float16,0,1.9796266555786133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,1,128,1,float16,fp8,0,2.4317760467529297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,2,128,1,float16,float16,0,1.9947466850280762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,2,128,1,float16,fp8,0,2.1906612714131675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,4,128,1,float16,float16,0,2.091466744740804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,64,8,128,1,float16,fp8,0,6.444538752237956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,8,128,1,float16,float16,0,2.01364803314209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,4,128,1,float16,fp8,0,2.132271925608317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,64,128,1,float16,float16,0,1.2154239813486736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,64,8,128,1,float16,fp8,0,2.152202606201172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,1,128,1,float16,float16,0,1.0184373060862224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,64,128,1,float16,fp8,0,1.2380692958831787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,1,128,1,float16,fp8,0,1.0596319834391277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,4,128,1,float16,float16,0,1.044981320699056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,2,128,1,float16,float16,0,1.277125358581543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,8,128,1,float16,float16,0,1.0388212998708088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,8,128,1,float16,fp8,0,1.105621337890625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,2,128,1,float16,fp8,0,1.1065706411997478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,64,4,128,1,float16,fp8,0,1.1148853302001953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,64,128,1,float16,float16,0,0.625925342241923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,1,128,1,float16,float16,0,0.5111200014750162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,64,128,1,float16,fp8,0,0.6620159943898519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,2,128,1,float16,float16,0,0.5239893198013306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,2,128,1,float16,fp8,0,0.5564746856689453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,1,128,1,float16,fp8,0,0.6217546860376993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,4,128,1,float16,float16,0,0.5428800185521444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,4,128,1,float16,fp8,0,0.5819840033849081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,8,128,1,float16,float16,0,0.533242662747701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,64,8,128,1,float16,fp8,0,0.5644479990005493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,1,128,1,float16,fp8,0,9.593701044718424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,2,128,1,float16,float16,0,10.00925318400065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,4,128,1,float16,float16,0,10.518917083740234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,1,128,1,float16,float16,0,10.043807983398438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,8,128,1,float16,float16,0,10.362154642740885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,4,128,1,float16,fp8,0,10.146474838256836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,8,128,1,float16,fp8,0,10.166266759236654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,64,2,128,1,float16,fp8,0,16.733450571695965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,64,128,1,float16,float16,0,6.052645365397136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,1,128,1,float16,float16,0,4.74235725402832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,64,128,1,float16,fp8,0,5.950357437133789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,1,128,1,float16,fp8,0,4.735434532165527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,2,128,1,float16,float16,0,4.989391962687175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,2,128,1,float16,fp8,0,4.787050565083821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,4,128,1,float16,float16,0,4.896037419637044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,64,128,1,float16,float16,0,2.943824132283529
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,64,128,1,float16,fp8,0,2.9975465138753257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,1,128,1,float16,float16,0,2.3130772908528647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,1,128,1,float16,fp8,0,2.3642080624898276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,2,128,1,float16,float16,0,2.365450700124105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,8,128,1,float16,float16,0,4.854122797648112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,8,128,1,float16,fp8,0,5.2476959228515625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,64,4,128,1,float16,fp8,0,9.264575958251953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,4,128,1,float16,float16,0,2.468832015991211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,2,128,1,float16,fp8,0,2.5071840286254883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,4,128,1,float16,fp8,0,2.5659306844075522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,8,128,1,float16,float16,0,2.3923412958780923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,64,128,1,float16,float16,0,1.5080374081929524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,64,128,1,float16,fp8,0,1.5258080164591472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,64,8,128,1,float16,fp8,0,2.588458697001139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,2,128,1,float16,float16,0,1.197109301884969
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,1,128,1,float16,float16,0,2.7982985178629556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,1,128,1,float16,fp8,0,1.2780906359354656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,2,128,1,float16,fp8,0,1.2289653619130452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,4,128,1,float16,float16,0,1.2497226397196453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,4,128,1,float16,fp8,0,1.3383679389953613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,8,128,1,float16,float16,0,1.2037333647410076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,64,8,128,1,float16,fp8,0,1.3202719688415527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,64,128,1,float16,float16,0,0.7612106800079346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,64,128,1,float16,fp8,0,0.8080586592356364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,1,128,1,float16,float16,0,0.5908693472544352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,2,128,1,float16,fp8,0,0.637882669766744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,4,128,1,float16,float16,0,0.6385546525319418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,1,128,1,float16,fp8,0,0.6425386667251587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,2,128,1,float16,float16,0,0.6118026574452718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,4,128,1,float16,fp8,0,0.6883839766184489
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,8,128,1,float16,float16,0,0.6254560152689616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,64,8,128,1,float16,fp8,0,0.6685547033945719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,64,128,1,float16,float16,0,0.393338680267334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,64,128,1,float16,fp8,0,0.4248266617457072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,2,128,1,float16,fp8,0,0.3405706485112508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,4,128,1,float16,float16,0,0.32897067070007324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,1,128,1,float16,float16,0,0.314463992913564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,8,128,1,float16,float16,0,0.3286133408546448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,1,128,1,float16,fp8,0,0.33990931510925293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,2,128,1,float16,float16,0,0.32400532563527423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,4,128,1,float16,fp8,0,0.35630400975545246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,64,8,128,1,float16,fp8,0,0.35809067885080975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,1,128,1,float16,float16,0,9.785141626993815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,1,128,1,float16,fp8,0,9.417296091715494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,2,128,1,float16,float16,0,10.231157302856445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,4,128,1,float16,float16,0,10.310789108276367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,8,128,1,float16,float16,0,10.122197469075521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,4,128,1,float16,fp8,0,10.290704091389975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,2,128,1,float16,fp8,0,9.512746810913086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,64,128,1,float16,float16,0,6.080181121826172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,1,128,1,float16,float16,0,4.42853323618571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,1,128,1,float16,fp8,0,4.848959922790527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,64,128,1,float16,fp8,0,6.252378463745117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,2,128,1,float16,float16,0,4.724922815958659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,2,128,1,float16,fp8,0,4.714223861694336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,4,128,1,float16,float16,0,4.9609066645304365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,64,128,1,float16,float16,0,3.048170725504557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,8,128,1,float16,float16,0,4.801797231038411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,1,128,1,float16,float16,0,2.21122137705485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,4,128,1,float16,fp8,0,5.312335968017578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,64,128,1,float16,fp8,0,3.1977545420328775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,64,8,128,1,float16,fp8,0,5.020458539326985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,1,128,1,float16,fp8,0,2.6449759801228843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,2,128,1,float16,float16,0,2.3308587074279785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,2,128,1,float16,fp8,0,2.531930605570475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,4,128,1,float16,float16,0,2.4353226025899253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,64,8,128,1,float16,fp8,0,26.890101114908855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,4,128,1,float16,fp8,0,2.666997273763021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,8,128,1,float16,float16,0,2.3988426526387534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,64,128,1,float16,float16,0,1.5381706555684407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,64,8,128,1,float16,fp8,0,2.5440425872802734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,1,128,1,float16,float16,0,1.1527040004730225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,64,128,1,float16,fp8,0,1.643994649251302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,1,128,1,float16,fp8,0,1.212154706319173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,2,128,1,float16,fp8,0,1.22270933787028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,4,128,1,float16,float16,0,1.2556320031483967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,2,128,1,float16,float16,0,2.347376028696696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,8,128,1,float16,float16,0,1.2050027052561443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,4,128,1,float16,fp8,0,1.3493226369222004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,64,8,128,1,float16,fp8,0,1.3219306468963623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,64,128,1,float16,float16,0,0.7885493437449137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,64,128,1,float16,fp8,0,0.8318666617075602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,1,128,1,float16,float16,0,0.5704319874445597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,1,128,1,float16,fp8,0,0.6435519854227701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,2,128,1,float16,float16,0,0.8606186707814535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,2,128,1,float16,fp8,0,0.626911997795105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,4,128,1,float16,float16,0,0.6343040068944296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,4,128,1,float16,fp8,0,0.6807200113932291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,8,128,1,float16,fp8,0,0.6743253072102865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,64,8,128,1,float16,float16,0,0.6182506481806437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,2,128,1,float16,float16,0,0.3144746621449788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,64,128,1,float16,float16,0,0.4022879997889201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,64,128,1,float16,fp8,0,0.4403146505355835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,1,128,1,float16,float16,0,0.31108800570170086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,1,128,1,float16,fp8,0,0.33260265986124676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,2,128,1,float16,fp8,0,0.33293867111206055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,4,128,1,float16,float16,0,0.32731199264526367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,4,128,1,float16,fp8,0,0.36401601632436115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,8,128,1,float16,float16,0,0.325546662012736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,64,8,128,1,float16,fp8,0,0.35133334000905353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,64,128,1,float16,float16,0,0.22060799598693848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,64,128,1,float16,fp8,0,0.23328532775243124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,1,128,1,float16,float16,0,0.16596266627311707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,2,128,1,float16,float16,0,0.16736533244450888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,1,128,1,float16,fp8,0,0.1785973310470581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,2,128,1,float16,fp8,0,0.18638400236765543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,4,128,1,float16,float16,0,0.1795466740926107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,4,128,1,float16,fp8,0,0.19768534104029337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,8,128,1,float16,float16,0,0.17737066745758057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,64,8,128,1,float16,fp8,0,0.19662400086720785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,1,128,1,float16,float16,0,5.728858947753906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,2,128,1,float16,float16,0,5.5596160888671875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,1,128,1,float16,fp8,0,5.843392054239909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,4,128,1,float16,float16,0,6.283450444539388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,8,128,1,float16,float16,0,5.9047896067301435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,4,128,1,float16,fp8,0,6.64797846476237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,2,128,1,float16,fp8,0,5.973258972167969
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,1,128,1,float16,float16,0,2.7000373204549155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,64,128,1,float16,float16,0,3.9364000956217446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,1,128,1,float16,fp8,0,3.0638294219970703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,2,128,1,float16,float16,0,2.8858559926350913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,64,128,1,float16,fp8,0,4.313925425211589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,2,128,1,float16,fp8,0,2.999301274617513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,64,8,128,1,float16,fp8,0,14.281754811604818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,4,128,1,float16,float16,0,3.115354537963867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,8,128,1,float16,float16,0,2.950144131978353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,64,128,1,float16,float16,0,1.9871892929077148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,64,128,1,float16,fp8,0,2.10042667388916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,4,128,1,float16,fp8,0,3.367216110229492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,1,128,1,float16,fp8,0,1.5362879435221355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,1,128,1,float16,float16,0,2.8831040064493814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,64,8,128,1,float16,fp8,0,3.26641050974528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,4,128,1,float16,float16,0,1.5185173352559407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,2,128,1,float16,float16,0,1.4328853289286296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,8,128,1,float16,float16,0,1.4682505925496419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,4,128,1,float16,fp8,0,1.7052586873372395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,64,128,1,float16,float16,0,0.9935466448465983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,2,128,1,float16,fp8,0,1.520330588022868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,64,128,1,float16,fp8,0,1.0792640050252278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,64,8,128,1,float16,fp8,0,2.486565272013346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,1,128,1,float16,float16,0,0.701205333073934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,1,128,1,float16,fp8,0,0.7568159898122152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,2,128,1,float16,fp8,0,0.7752213478088379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,2,128,1,float16,float16,0,0.7372693220774332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,4,128,1,float16,fp8,0,0.8581173419952393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,4,128,1,float16,float16,0,0.7802720069885254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,8,128,1,float16,fp8,0,0.8473280270894369
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,64,8,128,1,float16,float16,0,0.7890720367431641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,64,128,1,float16,float16,0,0.5053066809972128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,64,128,1,float16,fp8,0,0.5575733184814453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,1,128,1,float16,float16,0,0.3620320161183675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,1,128,1,float16,fp8,0,0.3992533286412557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,2,128,1,float16,fp8,0,0.41063467661539715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,2,128,1,float16,float16,0,0.3776266574859619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,4,128,1,float16,float16,0,0.40803198019663495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,8,128,1,float16,float16,0,0.38685866196950275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,8,128,1,float16,fp8,0,0.43355735143025714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,64,128,1,float16,float16,0,0.26976533730824787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,64,128,1,float16,fp8,0,0.295199990272522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,64,4,128,1,float16,fp8,0,0.4402346611022949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,1,128,1,float16,float16,0,0.19153600931167603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,8,128,1,float16,float16,0,0.21023466189702353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,1,128,1,float16,fp8,0,0.21634133656819662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,2,128,1,float16,float16,0,0.2011786699295044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,64,128,1,float16,fp8,0,0.16299200057983398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,2,128,1,float16,fp8,0,0.21649599075317383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,4,128,1,float16,float16,0,0.21286400159200033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,4,128,1,float16,fp8,0,0.23753599325815836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,2,128,1,float16,float16,0,0.11116266250610352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,64,128,1,float16,float16,0,0.14824000000953674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,64,8,128,1,float16,fp8,0,0.23246934016545615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,1,128,1,float16,float16,0,0.10840533177057902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,1,128,1,float16,fp8,0,0.11852799852689107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,2,128,1,float16,fp8,0,0.1246560017267863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,4,128,1,float16,float16,0,0.12060800194740295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,4,128,1,float16,fp8,0,0.1307199994723002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,8,128,1,float16,float16,0,0.116047998269399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,64,8,128,1,float16,fp8,0,0.12870933612187704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,1,128,1,float16,fp8,0,6.218640009562175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,8,128,1,float16,float16,0,6.152565638224284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,1,128,1,float16,float16,0,5.716543833414714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,8,128,1,float16,fp8,0,6.9409440358479815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,2,128,1,float16,fp8,0,6.3993174235026045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,4,128,1,float16,float16,0,6.488501230875651
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,1,128,1,float16,float16,0,2.888906796773275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,64,128,1,float16,float16,0,4.389247894287109
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,4,128,1,float16,fp8,0,7.310629526774089
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,1,128,1,float16,fp8,0,3.1234827041625977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,64,128,1,float16,fp8,0,4.951253255208333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,2,128,1,float16,float16,0,2.8972107569376626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,64,2,128,1,float16,float16,0,11.854517618815104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,2,128,1,float16,fp8,0,3.2170985539754233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,4,128,1,float16,float16,0,3.209493319193522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,64,128,1,float16,float16,0,2.198421319325765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,8,128,1,float16,fp8,0,3.41050656636556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,4,128,1,float16,fp8,0,3.6731465657552085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,1,128,1,float16,fp8,0,1.5774933497111003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,64,128,1,float16,fp8,0,2.432213306427002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,1,128,1,float16,float16,0,2.21014404296875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,2,128,1,float16,float16,0,1.4741387367248535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,2,128,1,float16,fp8,0,1.6285653114318848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,64,8,128,1,float16,float16,0,3.0927521387736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,4,128,1,float16,float16,0,1.6099306742350261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,4,128,1,float16,fp8,0,1.846981366475423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,8,128,1,float16,float16,0,1.5609653790791829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,64,128,1,float16,float16,0,1.1679306825002034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,64,8,128,1,float16,fp8,0,1.757754643758138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,2,128,1,float16,float16,0,0.752458651860555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,64,128,1,float16,fp8,0,1.2356639703114827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,2,128,1,float16,fp8,0,0.8299252986907959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,1,128,1,float16,float16,0,0.7402400175730387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,1,128,1,float16,fp8,0,0.8170186678568522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,4,128,1,float16,float16,0,0.8208853403727213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,8,128,1,float16,float16,0,0.7864800294240316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,1,128,1,float16,float16,0,0.3822773297627767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,4,128,1,float16,fp8,0,0.9261066913604736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,64,8,128,1,float16,fp8,0,0.8827146689097086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,64,128,1,float16,float16,0,0.5796800057093302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,64,128,1,float16,fp8,0,0.6387519836425781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,1,128,1,float16,fp8,0,0.4247200091679891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,2,128,1,float16,float16,0,0.3911626736323039
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,2,128,1,float16,fp8,0,0.43237332503000897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,8,128,1,float16,float16,0,0.4208213488260905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,4,128,1,float16,float16,0,0.42980798085530597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,64,128,1,float16,fp8,0,0.34015464782714844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,4,128,1,float16,fp8,0,0.4718720118204753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,64,8,128,1,float16,fp8,0,0.4634559949239095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,64,128,1,float16,float16,0,0.30024532477060956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,1,128,1,float16,float16,0,0.20081599553426108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,2,128,1,float16,float16,0,0.20761599143346152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,2,128,1,float16,fp8,0,0.23400000731150308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,1,128,1,float16,fp8,0,0.22614399592081705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,4,128,1,float16,float16,0,0.22858132918675741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,64,128,1,float16,float16,0,0.1616266667842865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,4,128,1,float16,fp8,0,0.2515466610590617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,8,128,1,float16,float16,0,0.22204800446828207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,64,8,128,1,float16,fp8,0,0.24815466006596884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,64,128,1,float16,fp8,0,0.1851573387781779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,1,128,1,float16,float16,0,0.1104906698067983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,4,128,1,float16,float16,0,0.12541332840919495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,1,128,1,float16,fp8,0,0.1258133351802826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,2,128,1,float16,fp8,0,0.12771733601888022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,2,128,1,float16,float16,0,0.11281599601109822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,4,128,1,float16,fp8,0,0.1397546629110972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,8,128,1,float16,float16,0,0.1206773320833842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,64,8,128,1,float16,fp8,0,0.13640000422795615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,2,128,1,float16,fp8,0,0.07050133248170216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,2,128,1,float16,float16,0,0.06419733166694641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,64,128,1,float16,float16,0,0.09199466307957967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,64,128,1,float16,fp8,0,0.10656533638636272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,1,128,1,float16,float16,0,0.05794133245944977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,1,128,1,float16,fp8,0,0.06598933537801106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,4,128,1,float16,float16,0,0.0698880006869634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,4,128,1,float16,fp8,0,0.07438933352629344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,8,128,1,float16,float16,0,0.06620266536871593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,64,8,128,1,float16,fp8,0,0.07513066629568736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,2,128,1,float16,float16,0,4.4084211985270185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,1,128,1,float16,float16,0,4.1815840403238935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,1,128,1,float16,fp8,0,4.756965319315593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,4,128,1,float16,float16,0,4.9549814860026045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,2,128,1,float16,fp8,0,4.857909202575684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,1,128,1,float16,float16,0,2.087792078653971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,64,128,1,float16,float16,0,3.60205872853597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,1,128,1,float16,fp8,0,2.3635147412618003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,64,128,1,float16,fp8,0,4.30836804707845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,4,128,1,float16,fp8,0,5.69273058573405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,2,128,1,float16,float16,0,2.1804800033569336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,2,128,1,float16,fp8,0,2.436181386311849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,8,128,1,float16,fp8,0,5.36674690246582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,4,128,1,float16,float16,0,2.487797260284424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,64,8,128,1,float16,float16,0,8.82044792175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,64,128,1,float16,float16,0,1.8121280670166016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,4,128,1,float16,fp8,0,2.858186721801758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,8,128,1,float16,fp8,0,2.6702934900919595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,64,8,128,1,float16,float16,0,2.332911968231201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,64,128,1,float16,fp8,0,2.146869341532389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,1,128,1,float16,fp8,0,1.1989493370056152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,2,128,1,float16,fp8,0,1.2353653113047283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,64,128,1,float16,float16,0,0.932090679804484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,4,128,1,float16,float16,0,1.2417386372884114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,8,128,1,float16,fp8,0,1.3683466911315918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,1,128,1,float16,float16,0,0.5392906665802002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,64,128,1,float16,fp8,0,1.0989332993825276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,8,128,1,float16,float16,0,1.173967997233073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,1,128,1,float16,float16,0,1.0622186660766602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,4,128,1,float16,fp8,0,1.4386720657348633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,1,128,1,float16,fp8,0,0.6096160014470419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,64,2,128,1,float16,float16,0,1.1221439838409424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,2,128,1,float16,float16,0,0.5742079814275106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,4,128,1,float16,fp8,0,0.7350719769795736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,4,128,1,float16,float16,0,0.6392000118891398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,2,128,1,float16,fp8,0,0.6312053203582764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,8,128,1,float16,float16,0,0.6068160136540731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,64,8,128,1,float16,fp8,0,0.6999093691507975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,64,128,1,float16,float16,0,0.4690186580022176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,64,128,1,float16,fp8,0,0.5598933299382528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,1,128,1,float16,float16,0,0.2789493401845296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,1,128,1,float16,fp8,0,0.31932799021402997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,2,128,1,float16,float16,0,0.2977973421414693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,2,128,1,float16,fp8,0,0.32840534051259357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,4,128,1,float16,float16,0,0.32844799757003784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,4,128,1,float16,fp8,0,0.37907199064890545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,1,128,1,float16,float16,0,0.1499733328819275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,8,128,1,float16,float16,0,0.31385066111882526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,64,128,1,float16,float16,0,0.24923733870188394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,64,128,1,float16,fp8,0,0.2944213350613912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,64,8,128,1,float16,fp8,0,0.3651466766993205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,1,128,1,float16,fp8,0,0.17088532447814941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,2,128,1,float16,float16,0,0.15760533014933267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,2,128,1,float16,fp8,0,0.1759893298149109
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,4,128,1,float16,float16,0,0.17299733559290567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,4,128,1,float16,fp8,0,0.1997226675351461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,8,128,1,float16,fp8,0,0.19416000445683798
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,64,8,128,1,float16,float16,0,0.16845866044362387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,64,128,1,float16,float16,0,0.13473066687583923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,64,128,1,float16,fp8,0,0.16218133767445883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,1,128,1,float16,float16,0,0.08340266346931458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,1,128,1,float16,fp8,0,0.09707199533780415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,2,128,1,float16,float16,0,0.08786666393280029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,2,128,1,float16,fp8,0,0.10079466303189595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,4,128,1,float16,float16,0,0.09528000156084697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,4,128,1,float16,fp8,0,0.11088533202807109
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,8,128,1,float16,float16,0,0.09203733007113139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,64,8,128,1,float16,fp8,0,0.10660800337791443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,64,128,1,float16,float16,0,0.07621333499749501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,2,128,1,float16,fp8,0,0.05495999753475189
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,64,128,1,float16,fp8,0,0.09215999643007915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,1,128,1,float16,float16,0,0.04588800172011057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,1,128,1,float16,fp8,0,0.05429866909980774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,2,128,1,float16,float16,0,0.04911466439565023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,64,128,1,float16,float16,0,0.04477333525816599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,4,128,1,float16,float16,0,0.05376533170541128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,8,128,1,float16,float16,0,0.04970133304595947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,4,128,1,float16,fp8,0,0.05862933397293091
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,2,128,1,float16,float16,0,0.029215998947620392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,64,8,128,1,float16,fp8,0,0.05726933479309082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,64,128,1,float16,fp8,0,0.05110933383305868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,1,128,1,float16,float16,0,0.026869334280490875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,8,128,1,float16,float16,0,0.03033600002527237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,1,128,1,float16,fp8,0,0.03278933217128118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,4,128,1,float16,float16,0,0.03086400032043457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,2,128,1,float16,fp8,0,0.034234667817751564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,4,128,1,float16,fp8,0,0.03542399903138479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,64,8,128,1,float16,fp8,0,0.03532266616821289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,1,128,1,float16,fp8,0,1.9974239667256672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,2,128,1,float16,float16,0,1.827221393585205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,2,128,1,float16,fp8,0,2.050858656565348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,1,128,1,float16,float16,0,1.7893493970235188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,4,128,1,float16,fp8,0,2.483834743499756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,64,128,1,float16,float16,0,1.6587413152058919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,1,128,1,float16,float16,0,0.9020853042602539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,8,128,1,float16,float16,0,2.0010239283243814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,64,128,1,float16,fp8,0,1.8463999430338542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,1,128,1,float16,fp8,0,1.0131253401438396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,2,128,1,float16,fp8,0,1.0406239827473958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,2,128,1,float16,float16,0,0.9404906431833903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,4,128,1,float16,float16,0,1.0705440044403076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,4,128,1,float16,float16,0,2.1226879755655923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,4,128,1,float16,fp8,0,1.254431962966919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,8,128,1,float16,float16,0,1.0062506993611653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,64,8,128,1,float16,fp8,0,1.190826654434204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,64,128,1,float16,float16,0,0.858191967010498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,1,128,1,float16,float16,0,0.46194132169087726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,64,128,1,float16,fp8,0,0.9366346995035807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,1,128,1,float16,fp8,0,0.517962654431661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,2,128,1,float16,float16,0,0.4799199899037679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,2,128,1,float16,fp8,0,0.5318666696548462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,4,128,1,float16,float16,0,0.5452213287353516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,8,128,1,float16,float16,0,0.5153173208236694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,1,128,1,float16,float16,0,0.24175999561945596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,8,128,1,float16,fp8,0,0.6110560099283854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,64,128,1,float16,fp8,0,0.4829813241958618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,64,4,128,1,float16,fp8,0,0.6419893503189087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,2,128,1,float16,fp8,0,0.27853333950042725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,64,8,128,1,float16,fp8,0,2.371551990509033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,64,128,1,float16,float16,0,0.4309866825739543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,4,128,1,float16,fp8,0,0.3320746620496114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,1,128,1,float16,fp8,0,0.27063467105229694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,2,128,1,float16,float16,0,0.24946133295694986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,4,128,1,float16,float16,0,0.2815200090408325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,8,128,1,float16,float16,0,0.26587732632954914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,64,8,128,1,float16,fp8,0,0.31547733147939044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,64,128,1,float16,fp8,0,0.25550933678944904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,1,128,1,float16,float16,0,0.13056000073750815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,1,128,1,float16,fp8,0,0.14687466621398926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,64,128,1,float16,float16,0,0.22685333093007407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,2,128,1,float16,float16,0,0.13391466935475668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,2,128,1,float16,fp8,0,0.15089600284894308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,4,128,1,float16,float16,0,0.15125866731007895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,4,128,1,float16,fp8,0,0.17499200503031412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,8,128,1,float16,float16,0,0.14169599612553915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,64,128,1,float16,float16,0,0.12167466680208842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,64,8,128,1,float16,fp8,0,0.1694773236910502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,64,128,1,float16,fp8,0,0.14115200440088907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,1,128,1,float16,float16,0,0.07373333474000295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,1,128,1,float16,fp8,0,0.08525866270065308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,2,128,1,float16,float16,0,0.07563200096289317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,2,128,1,float16,fp8,0,0.08685866991678874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,4,128,1,float16,float16,0,0.08269333342711131
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,4,128,1,float16,fp8,0,0.09754133224487305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,8,128,1,float16,float16,0,0.07814933359622955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,64,8,128,1,float16,fp8,0,0.09381332993507385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,64,128,1,float16,float16,0,0.07016000151634216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,64,128,1,float16,fp8,0,0.08230400085449219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,1,128,1,float16,float16,0,0.04058133314053217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,1,128,1,float16,fp8,0,0.04571199913819631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,8,128,1,float16,fp8,0,0.04827199876308441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,2,128,1,float16,float16,0,0.04163199911514918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,2,128,1,float16,fp8,0,0.04776533444722494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,4,128,1,float16,float16,0,0.0454773356517156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,4,128,1,float16,fp8,0,0.05277866621812185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,64,8,128,1,float16,float16,0,0.04133866727352142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,64,128,1,float16,float16,0,0.03988266736268997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,4,128,1,float16,float16,0,0.025199999411900837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,64,128,1,float16,fp8,0,0.04481600224971771
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,1,128,1,float16,float16,0,0.02404800057411194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,1,128,1,float16,fp8,0,0.02828266719977061
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,2,128,1,float16,float16,0,0.024160000185171764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,2,128,1,float16,fp8,0,0.029109333952267964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,4,128,1,float16,fp8,0,0.029696000119050343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,8,128,1,float16,float16,0,0.025061334172884624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,1,128,1,float16,fp8,0,0.020351999749739964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,64,8,128,1,float16,fp8,0,0.029103999336560566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,64,128,1,float16,float16,0,0.019530666371186573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,64,128,1,float16,fp8,0,0.02439466615517934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,1,128,1,float16,float16,0,0.016730666160583496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,2,128,1,float16,float16,0,0.017130666722853977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,2,128,1,float16,fp8,0,0.019088000059127808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,4,128,1,float16,float16,0,0.016778666526079178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,4,128,1,float16,fp8,0,0.019653332730134327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,8,128,1,float16,float16,0,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,64,8,128,1,float16,fp8,0,0.019573333362738293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,1,128,1,float16,float16,0,0.8384319941202799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,1,128,1,float16,fp8,0,0.8948853015899658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,2,128,1,float16,float16,0,0.9065013726552328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,2,128,1,float16,fp8,0,0.9440906842549642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,4,128,1,float16,float16,0,1.0380160013834636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,4,128,1,float16,fp8,0,1.155349334081014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,8,128,1,float16,float16,0,0.9979466597239176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,64,128,1,float16,float16,0,0.8025386333465576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,64,8,128,1,float16,fp8,0,1.031605323155721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,1,128,1,float16,float16,0,0.4262666702270508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,64,128,1,float16,fp8,0,0.8467733065287272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,1,128,1,float16,fp8,0,0.4607786734898885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,2,128,1,float16,float16,0,0.4612319866816203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,4,128,1,float16,float16,0,0.5301546653111776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,2,128,1,float16,fp8,0,0.4864906469980876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,8,128,1,float16,float16,0,0.5012853145599365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,64,128,1,float16,float16,0,0.41147732734680176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,64,128,1,float16,fp8,0,0.438101331392924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,4,128,1,float16,fp8,0,0.5837173461914062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,64,8,128,1,float16,fp8,0,0.5231893459955851
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,1,128,1,float16,float16,0,0.22118399540583292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,1,128,1,float16,fp8,0,0.24423466126124063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,2,128,1,float16,float16,0,0.24032533168792725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,2,128,1,float16,fp8,0,0.25628799200057983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,4,128,1,float16,fp8,0,0.304751992225647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,4,128,1,float16,float16,0,0.26969067255655926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,8,128,1,float16,float16,0,0.26190932591756183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,64,8,128,1,float16,fp8,0,0.2757386763890584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,64,128,1,float16,float16,0,0.2199946641921997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,2,128,1,float16,fp8,0,0.14232533176740012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,64,128,1,float16,fp8,0,0.23665066560109457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,1,128,1,float16,float16,0,0.11938666303952535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,1,128,1,float16,fp8,0,0.136543999115626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,2,128,1,float16,float16,0,0.12805333733558655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,4,128,1,float16,float16,0,0.14341333508491516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,4,128,1,float16,fp8,0,0.16532267133394876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,8,128,1,float16,float16,0,0.13821867108345032
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,64,128,1,float16,float16,0,0.12040000160535176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,64,8,128,1,float16,fp8,0,0.1513706644376119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,4,128,1,float16,float16,0,0.07924800117810567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,64,128,1,float16,fp8,0,0.12709866960843405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,1,128,1,float16,float16,0,0.06825066606203715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,2,128,1,float16,float16,0,0.07121600210666656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,4,128,1,float16,fp8,0,0.0929813285668691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,8,128,1,float16,fp8,0,0.08413867155710857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,64,128,1,float16,float16,0,0.06712000072002411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,1,128,1,float16,float16,0,0.03482666611671448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,1,128,1,float16,fp8,0,0.07794666786988576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,2,128,1,float16,fp8,0,0.08080533146858215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,64,8,128,1,float16,float16,0,0.07247999807198842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,64,128,1,float16,fp8,0,0.07155199845631917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,2,128,1,float16,float16,0,0.036506667733192444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,1,128,1,float16,fp8,0,0.03844800094763438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,2,128,1,float16,fp8,0,0.03938133269548416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,4,128,1,float16,float16,0,0.03974399964014689
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,4,128,1,float16,fp8,0,0.043680002291997276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,8,128,1,float16,float16,0,0.03651199986537298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,64,8,128,1,float16,fp8,0,0.041434665520985924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,64,128,1,float16,float16,0,0.03771200031042099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,64,128,1,float16,fp8,0,0.03870933254559835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,1,128,1,float16,float16,0,0.0215786670645078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,1,128,1,float16,fp8,0,0.024197332561016083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,2,128,1,float16,float16,0,0.02110933264096578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,2,128,1,float16,fp8,0,0.024512000381946564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,64,128,1,float16,fp8,0,0.020346666375796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,1,128,1,float16,float16,0,0.015194666882356008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,4,128,1,float16,float16,0,0.023024000227451324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,4,128,1,float16,fp8,0,0.02605866640806198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,8,128,1,float16,float16,0,0.022624000906944275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,64,8,128,1,float16,fp8,0,0.025519999365011852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,64,128,1,float16,float16,0,0.01844266677896182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,1,128,1,float16,fp8,0,0.01803733284274737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,2,128,1,float16,float16,0,0.015552000453074774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,2,128,1,float16,fp8,0,0.01782400036851565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,4,128,1,float16,float16,0,0.015893333901961643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,4,128,1,float16,fp8,0,0.01806933308641116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,8,128,1,float16,float16,0,0.015610666324694952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,64,8,128,1,float16,fp8,0,0.017717332889636356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,64,128,1,float16,fp8,0,0.01874133323629697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,64,128,1,float16,float16,0,0.016293333222468693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,1,128,1,float16,float16,0,0.014959999670584997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,1,128,1,float16,fp8,0,0.01743999992807706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,2,128,1,float16,float16,0,0.015274666249752045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,2,128,1,float16,fp8,0,0.017344000438849132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,4,128,1,float16,float16,0,0.01523200049996376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,4,128,1,float16,fp8,0,0.017514667163292568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,8,128,1,float16,float16,0,0.014853333433469137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,64,8,128,1,float16,fp8,0,0.017450666675964992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,1,128,1,float16,float16,0,0.42662398020426434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,1,128,1,float16,fp8,0,0.4571946859359741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,8,128,1,float16,fp8,0,0.5202666521072388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,2,128,1,float16,float16,0,0.46378131707509357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,4,128,1,float16,float16,0,0.5210346778233846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,64,128,1,float16,float16,0,0.46352001031239826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,2,128,1,float16,fp8,0,0.48738666375478107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,4,128,1,float16,fp8,0,0.5878719886144003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,1,128,1,float16,float16,0,0.2223680019378662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,64,8,128,1,float16,float16,0,0.5017226537068685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,1,128,1,float16,fp8,0,0.24562132358551025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,4,128,1,float16,fp8,0,0.30595733722050983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,8,128,1,float16,float16,0,0.2606239914894104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,2,128,1,float16,float16,0,0.23548799753189087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,4,128,1,float16,float16,0,0.2701920072237651
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,64,128,1,float16,float16,0,0.24016533295313516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,64,128,1,float16,fp8,0,0.23002666234970093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,2,128,1,float16,fp8,0,0.25699732700983685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,1,128,1,float16,fp8,0,0.13766933480898538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,8,128,1,float16,fp8,0,0.27447466055552167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,1,128,1,float16,float16,0,0.11965333422025044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,2,128,1,float16,float16,0,0.12764267126719156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,8,128,1,float16,float16,0,0.13542933265368143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,2,128,1,float16,fp8,0,0.1428053379058838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,64,64,128,1,float16,fp8,0,0.444048007329305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,8,128,1,float16,fp8,0,0.149125337600708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,4,128,1,float16,float16,0,0.1441386640071869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,64,128,1,float16,float16,0,0.1283199985822042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,64,4,128,1,float16,fp8,0,0.16671999295552573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,2,128,1,float16,fp8,0,0.08187733093897502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,4,128,1,float16,float16,0,0.08020799855391185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,64,128,1,float16,fp8,0,0.12131200234095256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,1,128,1,float16,float16,0,0.06701866785685222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,8,128,1,float16,float16,0,0.07096533477306366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,1,128,1,float16,fp8,0,0.07840533554553986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,64,128,1,float16,fp8,0,0.06885333359241486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,2,128,1,float16,float16,0,0.07073600093523662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,1,128,1,float16,fp8,0,0.037392000357309975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,4,128,1,float16,fp8,0,0.09178133805592854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,64,8,128,1,float16,fp8,0,0.08362133304278056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,64,128,1,float16,float16,0,0.07146133482456207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,1,128,1,float16,float16,0,0.03398933261632919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,2,128,1,float16,float16,0,0.03730133424202601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,2,128,1,float16,fp8,0,0.0401653324564298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,4,128,1,float16,float16,0,0.04250133534272512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,4,128,1,float16,fp8,0,0.04530133306980133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,8,128,1,float16,float16,0,0.03703466554482778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,64,8,128,1,float16,fp8,0,0.04127999891837438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,64,128,1,float16,float16,0,0.04036800066630045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,64,128,1,float16,fp8,0,0.033813332517941795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,1,128,1,float16,float16,0,0.0215786670645078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,1,128,1,float16,fp8,0,0.02425066630045573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,2,128,1,float16,float16,0,0.021018666525681812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,2,128,1,float16,fp8,0,0.024319998919963837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,64,128,1,float16,fp8,0,0.019306667149066925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,4,128,1,float16,float16,0,0.02292799949645996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,4,128,1,float16,fp8,0,0.02585600068171819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,8,128,1,float16,float16,0,0.022474666436513264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,64,8,128,1,float16,fp8,0,0.025392000873883564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,64,128,1,float16,float16,0,0.01932799940307935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,1,128,1,float16,float16,0,0.01523200049996376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,8,128,1,float16,float16,0,0.015546667079130808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,1,128,1,float16,fp8,0,0.0179626668492953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,2,128,1,float16,float16,0,0.015295999745527903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,2,128,1,float16,fp8,0,0.017909333109855652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,4,128,1,float16,float16,0,0.01579733317097028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,4,128,1,float16,fp8,0,0.018016000588734944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,64,8,128,1,float16,fp8,0,0.01759999990463257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,64,128,1,float16,fp8,0,0.015008000036080679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,64,128,1,float16,float16,0,0.014287999520699183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,1,128,1,float16,float16,0,0.014661333213249842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,1,128,1,float16,fp8,0,0.017429333180189133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,8,128,1,float16,fp8,0,0.017231999586025875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,2,128,1,float16,float16,0,0.014991999914248785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,2,128,1,float16,fp8,0,0.01758933315674464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,4,128,1,float16,float16,0,0.014917333920796713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,1,128,1,float16,fp8,0,0.01741333305835724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,4,128,1,float16,fp8,0,0.017664000391960144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,64,8,128,1,float16,float16,0,0.015253332753976187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,64,128,1,float16,float16,0,0.013631999492645264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,64,128,1,float16,fp8,0,0.014560000350077948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,1,128,1,float16,float16,0,0.014485333114862442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,2,128,1,float16,float16,0,0.014864000181357065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,2,128,1,float16,fp8,0,0.017279999951521557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,4,128,1,float16,float16,0,0.014789332946141561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,4,128,1,float16,fp8,0,0.017535999417304993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,8,128,1,float16,float16,0,0.014405333747466406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,64,8,128,1,float16,fp8,0,0.017152000218629837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,1,128,1,float16,float16,0,0.22100265820821127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,1,128,1,float16,fp8,0,0.24411733945210776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,2,128,1,float16,float16,0,0.23585599660873413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,2,128,1,float16,fp8,0,0.26070932547251385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,4,128,1,float16,float16,0,0.26791467269261676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,4,128,1,float16,fp8,0,0.3063039978345235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,1,128,1,float16,float16,0,0.12069867054621379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,8,128,1,float16,float16,0,0.25411732991536456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,64,128,1,float16,float16,0,0.338154673576355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,64,128,1,float16,fp8,0,0.32020266850789386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,1,128,1,float16,fp8,0,0.13743999600410461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,2,128,1,float16,float16,0,0.1264693339665731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,64,8,128,1,float16,fp8,0,0.2752319971720378
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,2,128,1,float16,fp8,0,0.14300266901652017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,4,128,1,float16,float16,0,0.14389866590499878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,8,128,1,float16,fp8,0,0.14941333731015524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,64,128,1,float16,float16,0,0.17643199364344278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,8,128,1,float16,float16,0,0.13500266273816428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,1,128,1,float16,float16,0,0.06864533325036366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,2,128,1,float16,float16,0,0.07252799967924754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,64,4,128,1,float16,fp8,0,0.16581867138544717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,1,128,1,float16,fp8,0,0.07786666850248973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,64,128,1,float16,fp8,0,0.1692053278287252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,2,128,1,float16,fp8,0,0.08159466584523518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,4,128,1,float16,float16,0,0.07969066500663757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,4,128,1,float16,fp8,0,0.0922933320204417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,8,128,1,float16,float16,0,0.07229866584142049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,64,8,128,1,float16,fp8,0,0.08355733752250671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,64,128,1,float16,float16,0,0.09462933739026387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,64,128,1,float16,fp8,0,0.09360532959302266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,1,128,1,float16,float16,0,0.0341386670867602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,1,128,1,float16,fp8,0,0.03788266579310099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,2,128,1,float16,float16,0,0.03607466568549474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,2,128,1,float16,fp8,0,0.03991466760635376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,4,128,1,float16,float16,0,0.0421973317861557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,4,128,1,float16,fp8,0,0.0444106658299764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,8,128,1,float16,float16,0,0.03617066641648611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,64,8,128,1,float16,fp8,0,0.041365332901477814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,64,128,1,float16,float16,0,0.052709331115086876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,4,128,1,float16,float16,0,0.02231466770172119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,2,128,1,float16,fp8,0,0.024469333390394848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,64,128,1,float16,fp8,0,0.047135998805363975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,1,128,1,float16,float16,0,0.021365332106749218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,2,128,1,float16,float16,0,0.02120000123977661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,1,128,1,float16,fp8,0,0.024314666787783306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,4,128,1,float16,fp8,0,0.025765334566434223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,8,128,1,float16,float16,0,0.02221333235502243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,64,8,128,1,float16,fp8,0,0.02571733295917511
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,2,128,1,float16,fp8,0,0.01783466711640358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,64,128,1,float16,float16,0,0.025349333882331848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,64,128,1,float16,fp8,0,0.02555199960867564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,1,128,1,float16,float16,0,0.015423999478419622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,1,128,1,float16,fp8,0,0.017898666361967724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,64,128,1,float16,float16,0,0.01752000053723653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,2,128,1,float16,float16,0,0.015509333461523056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,4,128,1,float16,float16,0,0.01540800059835116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,4,128,1,float16,fp8,0,0.017850667238235474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,8,128,1,float16,float16,0,0.015749332805474598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,64,8,128,1,float16,fp8,0,0.01770666614174843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,64,128,1,float16,fp8,0,0.01850133389234543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,1,128,1,float16,float16,0,0.014581333845853806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,1,128,1,float16,fp8,0,0.01718933383623759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,2,128,1,float16,float16,0,0.015311999867359797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,2,128,1,float16,fp8,0,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,4,128,1,float16,float16,0,0.014842666685581207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,4,128,1,float16,fp8,0,0.01757866640885671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,8,128,1,float16,float16,0,0.014778666198253632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,64,8,128,1,float16,fp8,0,0.016858667135238647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,64,128,1,float16,float16,0,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,64,128,1,float16,fp8,0,0.014522666732470194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,1,128,1,float16,float16,0,0.014362666755914688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,1,128,1,float16,fp8,0,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,2,128,1,float16,float16,0,0.014645333091417948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,2,128,1,float16,fp8,0,0.017008000363906223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,4,128,1,float16,float16,0,0.014741333822409311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,4,128,1,float16,fp8,0,0.017125333348910015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,8,128,1,float16,float16,0,0.014165333161751429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,64,8,128,1,float16,fp8,0,0.016565332810084026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,64,128,1,float16,float16,0,0.013237333546082178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,64,128,1,float16,fp8,0,0.014117332796255747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,1,128,1,float16,float16,0,0.013712000101804733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,1,128,1,float16,fp8,0,0.01595199977358182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,2,128,1,float16,float16,0,0.013861333330472311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,2,128,1,float16,fp8,0,0.015861333658297855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,4,128,1,float16,float16,0,0.013855999956528345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,4,128,1,float16,fp8,0,0.01579733317097028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,8,128,1,float16,float16,0,0.013776000589132309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,64,8,128,1,float16,fp8,0,0.016794666647911072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,1,128,1,float16,float16,0,0.11875200271606445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,1,128,1,float16,fp8,0,0.13678399721781412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,2,128,1,float16,fp8,0,0.14441600441932678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,4,128,1,float16,float16,0,0.14291200041770935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,64,128,1,float16,float16,0,0.2795253396034241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,2,128,1,float16,float16,0,0.12737600008646646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,64,128,1,float16,fp8,0,0.2645813425381978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,4,128,1,float16,fp8,0,0.16553599635759988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,8,128,1,float16,float16,0,0.13185600439707437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,64,8,128,1,float16,fp8,0,0.14747732877731323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,1,128,1,float16,float16,0,0.06854933500289917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,1,128,1,float16,fp8,0,0.07884799937407176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,2,128,1,float16,fp8,0,0.08148266871770223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,2,128,1,float16,float16,0,0.07200533151626587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,4,128,1,float16,float16,0,0.07983999947706859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,8,128,1,float16,fp8,0,0.08203200002511342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,64,128,1,float16,float16,0,0.1460693379243215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,4,128,1,float16,fp8,0,0.09197333455085754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,2,128,1,float16,float16,0,0.03786666691303253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,1,128,1,float16,fp8,0,0.04140799989302953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,64,8,128,1,float16,float16,0,0.07063466807206471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,64,128,1,float16,fp8,0,0.14094932874043783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,1,128,1,float16,float16,0,0.03448000053564707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,2,128,1,float16,fp8,0,0.039093332986036934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,4,128,1,float16,fp8,0,0.04343999922275543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,64,128,1,float16,float16,0,0.0775626649459203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,4,128,1,float16,float16,0,0.04177600145339966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,8,128,1,float16,float16,0,0.03624533365170161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,64,8,128,1,float16,fp8,0,0.04114133367935816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,64,128,1,float16,fp8,0,0.07165333131949107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,1,128,1,float16,float16,0,0.02067733307679494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,1,128,1,float16,fp8,0,0.024085332949956257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,2,128,1,float16,float16,0,0.02102400114138921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,2,128,1,float16,fp8,0,0.024288001159826916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,4,128,1,float16,float16,0,0.022831998765468597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,4,128,1,float16,fp8,0,0.025936000049114227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,8,128,1,float16,float16,0,0.021850667893886566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,64,8,128,1,float16,fp8,0,0.025248001019159954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,64,128,1,float16,float16,0,0.03770133356253306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,64,128,1,float16,fp8,0,0.03746666759252548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,1,128,1,float16,float16,0,0.015418666104475657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,1,128,1,float16,fp8,0,0.01815466706951459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,2,128,1,float16,float16,0,0.015189333508412043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,2,128,1,float16,fp8,0,0.01781333362062772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,4,128,1,float16,float16,0,0.01543466622630755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,4,128,1,float16,fp8,0,0.01791999985774358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,1,128,1,float16,float16,0,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,8,128,1,float16,float16,0,0.015397333850463232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,64,8,128,1,float16,fp8,0,0.017488000293572743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,64,128,1,float16,float16,0,0.02380266785621643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,64,128,1,float16,fp8,0,0.024442667762438457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,1,128,1,float16,fp8,0,0.0174346665541331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,2,128,1,float16,float16,0,0.014848000059525171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,2,128,1,float16,fp8,0,0.01754133279124896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,4,128,1,float16,float16,0,0.014954666296641031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,4,128,1,float16,fp8,0,0.01739199956258138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,8,128,1,float16,fp8,0,0.01676799977819125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,64,8,128,1,float16,float16,0,0.015125333021084467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,64,128,1,float16,float16,0,0.01684800038735072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,2,128,1,float16,float16,0,0.014661333213249842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,64,128,1,float16,fp8,0,0.017957333475351334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,1,128,1,float16,float16,0,0.01471466695268949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,1,128,1,float16,fp8,0,0.017504000415404636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,2,128,1,float16,fp8,0,0.016976000120242436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,4,128,1,float16,float16,0,0.014965333044528961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,4,128,1,float16,fp8,0,0.017082666357358296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,1,128,1,float16,fp8,0,0.01578666642308235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,8,128,1,float16,float16,0,0.014298666268587112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,64,8,128,1,float16,fp8,0,0.01661866654952367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,64,128,1,float16,float16,0,0.013525333255529404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,64,128,1,float16,fp8,0,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,1,128,1,float16,float16,0,0.01393066719174385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,2,128,1,float16,float16,0,0.014682666709025701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,2,128,1,float16,fp8,0,0.015861333658297855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,4,128,1,float16,float16,0,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,4,128,1,float16,fp8,0,0.017114666601022083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,8,128,1,float16,float16,0,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,64,8,128,1,float16,fp8,0,0.01553600033124288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,2,128,1,float16,fp8,0,0.01570133368174235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,64,128,1,float16,float16,0,0.01321600005030632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,64,128,1,float16,fp8,0,0.014416000495354334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,1,128,1,float16,float16,0,0.013999999811251959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,1,128,1,float16,fp8,0,0.015647999942302704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,2,128,1,float16,float16,0,0.013573333621025085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,4,128,1,float16,float16,0,0.01379199946920077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,4,128,1,float16,fp8,0,0.01586666703224182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,8,128,1,float16,float16,0,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,64,8,128,1,float16,fp8,0,0.015263999501864115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,0,0.02422933280467987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,0,0.025770666698614757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,0,0.03032533327738444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,0,0.03224000086386999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,0,0.043466667334238686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,0,0.04760533571243286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,0,0.06736533343791962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,64,128,1,float16,float16,0,0.21761600176493326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,64,128,1,float16,fp8,0,0.22587200005849203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,0,0.07108266651630402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,0,0.015392000476519266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,0,0.016778666526079178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,0,0.01886933296918869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,0,0.020314666132132213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,0,0.04005866746107737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,0,0.025285333395004272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,0,0.02770666778087616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,0,0.03782399992148081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,64,128,1,float16,float16,0,0.11357866724332173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,64,128,1,float16,fp8,0,0.11920000116030376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,0,0.012853333105643591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,0,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,0,0.013173333058754602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,0,0.014458666245142618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,0,0.01695999999841054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,64,128,1,float16,fp8,0,0.067071999112765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,0,0.01836266616980235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,0,0.023402666052182514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,0,0.025983999172846477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,64,128,1,float16,float16,0,0.06322133541107178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,0,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,0,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,0,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,0,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,0,0.012666666259368261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,0,0.013781332721312841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,0,0.016250666230916977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,0,0.01735466718673706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,64,128,1,float16,float16,0,0.03669866671164831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,64,128,1,float16,fp8,0,0.04001600046952566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,0,0.01138666644692421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,0,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,0,0.011706666400035223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,0,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,0,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,64,128,1,float16,fp8,0,0.02588266630967458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,0,0.012949333836634954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,0,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,0,0.012960000584522883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,64,128,1,float16,float16,0,0.024010665714740753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,0,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,0,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,0,0.011365332951148352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,0,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,0,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,0,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,0,0.012725333372751871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,0,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,64,128,1,float16,float16,0,0.016677333662907284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,64,128,1,float16,fp8,0,0.018415999909241993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,0,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,0,0.011616000284751257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,0,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,64,128,1,float16,float16,0,0.013013333082199097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,0,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,0,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,0,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,0,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,0,0.011605333536863327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,0,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,0,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,64,128,1,float16,fp8,0,0.014853333433469137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,0,0.010944000134865442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,64,128,1,float16,float16,0,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,0,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,0,0.01249066616098086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,0,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,0,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,0,0.011621333658695221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,0,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,64,128,1,float16,fp8,0,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,0,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,0,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,0,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,0,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,0,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,0,0.01128000020980835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,0,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,1,128,1,float16,fp8,0,40.80276743570963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,1,128,1,float16,float16,0,45.41381327311198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,4,128,1,float16,fp8,0,41.71997833251953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,2,128,1,float16,float16,0,44.9463857014974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,8,128,1,float16,fp8,0,41.836201985677086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,4,128,1,float16,float16,0,46.56834920247396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,8,128,1,float16,float16,0,45.931437174479164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,48,128,1,float16,float16,0,23.901349385579426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,1,128,1,float16,fp8,0,20.4464848836263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,48,128,1,float16,fp8,0,21.8275146484375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,1,128,1,float16,float16,0,23.30046844482422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,2,128,1,float16,float16,0,24.032997131347656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,2,128,1,float16,fp8,0,20.57482655843099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,48,2,128,1,float16,fp8,0,72.77466328938802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,4,128,1,float16,float16,0,25.315963745117188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,48,128,1,float16,fp8,0,12.23593012491862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,48,128,1,float16,float16,0,14.24413808186849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,1,128,1,float16,float16,0,15.012437184651693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,2,128,1,float16,float16,0,12.693951924641928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,4,128,1,float16,fp8,0,21.72724151611328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,8,128,1,float16,fp8,0,21.66699727376302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,1,128,1,float16,fp8,0,19.833296457926433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,48,8,128,1,float16,float16,0,24.541114807128906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,2,128,1,float16,fp8,0,11.5263303120931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,4,128,1,float16,fp8,0,10.550949096679688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,4,128,1,float16,float16,0,12.293951670328775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,48,128,1,float16,float16,0,6.379146575927734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,8,128,1,float16,float16,0,11.864315032958984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,1,128,1,float16,float16,0,5.4966080983479815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,1,128,1,float16,fp8,0,5.213941256205241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,48,8,128,1,float16,fp8,0,10.598976135253906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,48,128,1,float16,fp8,0,9.641541163126627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,2,128,1,float16,float16,0,5.954154968261719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,2,128,1,float16,fp8,0,5.559455871582031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,4,128,1,float16,float16,0,5.719615936279297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,4,128,1,float16,fp8,0,5.539679845174153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,8,128,1,float16,float16,0,5.778485616048177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,48,8,128,1,float16,fp8,0,5.333168029785156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,1,128,1,float16,float16,0,26.01970672607422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,2,128,1,float16,fp8,0,23.52716827392578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,2,128,1,float16,float16,0,26.349088033040363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,4,128,1,float16,fp8,0,24.09820810953776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,4,128,1,float16,float16,0,26.615193684895832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,8,128,1,float16,fp8,0,24.246991475423176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,8,128,1,float16,float16,0,26.205034891764324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,48,128,1,float16,fp8,0,12.762235005696615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,48,128,1,float16,float16,0,14.112037658691406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,1,128,1,float16,float16,0,13.959349314371744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,1,128,1,float16,fp8,0,11.73532231648763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,2,128,1,float16,float16,0,13.335562388102213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,2,128,1,float16,fp8,0,11.799087524414062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,48,1,128,1,float16,fp8,0,44.40918477376302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,1,128,1,float16,float16,0,6.454410552978516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,48,128,1,float16,float16,0,7.615231831868489
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,48,128,1,float16,fp8,0,6.564655939737956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,8,128,1,float16,fp8,0,12.035621643066406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,4,128,1,float16,float16,0,14.15225092569987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,4,128,1,float16,fp8,0,12.230842590332031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,48,8,128,1,float16,float16,0,14.298768361409506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,1,128,1,float16,fp8,0,10.19816525777181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,2,128,1,float16,float16,0,6.589914957682292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,2,128,1,float16,fp8,0,6.103589375813802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,4,128,1,float16,float16,0,6.557573318481445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,48,128,1,float16,float16,0,3.771018664042155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,4,128,1,float16,fp8,0,6.102965037027995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,1,128,1,float16,float16,0,3.137749354044596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,8,128,1,float16,fp8,0,6.093269348144531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,48,8,128,1,float16,float16,0,6.643930435180664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,1,128,1,float16,fp8,0,3.006058692932129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,48,128,1,float16,fp8,0,5.253685315450032
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,2,128,1,float16,float16,0,3.0889228185017905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,2,128,1,float16,fp8,0,3.222581227620443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,4,128,1,float16,float16,0,3.0964533487955728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,4,128,1,float16,fp8,0,3.230479876200358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,8,128,1,float16,float16,0,3.133962631225586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,48,8,128,1,float16,fp8,0,3.6267627080281577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,1,128,1,float16,float16,0,18.772176106770832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,2,128,1,float16,fp8,0,16.839861551920574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,2,128,1,float16,float16,0,18.55780792236328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,4,128,1,float16,fp8,0,17.048362731933594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,4,128,1,float16,float16,0,19.351802825927734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,8,128,1,float16,float16,0,18.549908955891926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,8,128,1,float16,fp8,0,17.259056091308594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,48,1,128,1,float16,fp8,0,28.308453877766926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,1,128,1,float16,fp8,0,8.240367889404297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,48,128,1,float16,fp8,0,9.119370778401693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,48,128,1,float16,float16,0,10.52243169148763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,1,128,1,float16,float16,0,9.333626429239908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,2,128,1,float16,fp8,0,8.334885279337565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,2,128,1,float16,float16,0,9.02241579691569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,4,128,1,float16,float16,0,9.90884780883789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,1,128,1,float16,float16,0,4.33024533589681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,48,128,1,float16,float16,0,4.853765487670898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,48,128,1,float16,fp8,0,4.973018646240234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,1,128,1,float16,fp8,0,4.320165316263835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,2,128,1,float16,float16,0,4.503727912902832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,8,128,1,float16,fp8,0,8.608512242635092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,8,128,1,float16,float16,0,9.615296045939127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,2,128,1,float16,fp8,0,4.703061421712239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,4,128,1,float16,float16,0,4.617978731791179
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,48,4,128,1,float16,fp8,0,14.754175821940104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,48,128,1,float16,fp8,0,2.618336041768392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,4,128,1,float16,fp8,0,4.33954652150472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,8,128,1,float16,float16,0,4.490357398986816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,48,128,1,float16,float16,0,2.4156479835510254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,1,128,1,float16,fp8,0,2.24837334950765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,48,8,128,1,float16,fp8,0,4.46615473429362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,1,128,1,float16,float16,0,2.1477279663085938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,2,128,1,float16,fp8,0,2.2049919764200845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,4,128,1,float16,float16,0,2.2035253842671714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,2,128,1,float16,float16,0,4.964687983194987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,8,128,1,float16,fp8,0,2.2437920570373535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,4,128,1,float16,fp8,0,2.281061331431071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,48,8,128,1,float16,float16,0,2.2314666112264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,1,128,1,float16,fp8,0,21.50439961751302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,1,128,1,float16,float16,0,23.6440912882487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,2,128,1,float16,fp8,0,21.659451802571613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,2,128,1,float16,float16,0,23.60210673014323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,4,128,1,float16,fp8,0,22.444747924804688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,8,128,1,float16,fp8,0,22.393269856770832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,8,128,1,float16,float16,0,24.409617106119793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,1,128,1,float16,fp8,0,10.783279418945312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,48,128,1,float16,float16,0,13.978399912516275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,1,128,1,float16,float16,0,12.350874582926432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,48,128,1,float16,fp8,0,12.151908874511719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,2,128,1,float16,fp8,0,10.880736033121744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,2,128,1,float16,float16,0,12.318837483723959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,4,128,1,float16,float16,0,12.266736348470053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,1,128,1,float16,float16,0,5.751898447672526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,48,128,1,float16,float16,0,6.144650777180989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,48,128,1,float16,fp8,0,6.108879725138347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,1,128,1,float16,fp8,0,5.8035093943278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,4,128,1,float16,fp8,0,11.232011159261068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,8,128,1,float16,fp8,0,11.2762082417806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,48,8,128,1,float16,float16,0,12.602261861165365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,2,128,1,float16,float16,0,5.970906575520833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,4,128,1,float16,float16,0,6.0957597096761065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,2,128,1,float16,fp8,0,5.778522491455078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,48,128,1,float16,float16,0,3.1394666035970054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,48,4,128,1,float16,float16,0,59.913777669270836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,4,128,1,float16,fp8,0,5.625957489013672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,48,128,1,float16,fp8,0,3.1465813318888345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,8,128,1,float16,float16,0,5.7999623616536455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,48,8,128,1,float16,fp8,0,5.675535837809245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,1,128,1,float16,float16,0,2.8656320571899414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,1,128,1,float16,fp8,0,2.818410555521647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,2,128,1,float16,float16,0,2.8067038853963218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,4,128,1,float16,float16,0,2.8544747034708657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,4,128,1,float16,fp8,0,2.872159957885742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,8,128,1,float16,float16,0,2.852970759073893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,2,128,1,float16,fp8,0,4.4079945882161455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,1,128,1,float16,float16,0,1.4268372853597004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,48,128,1,float16,float16,0,1.576357364654541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,48,128,1,float16,fp8,0,1.767258644104004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,1,128,1,float16,fp8,0,1.4127732912699382
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,48,8,128,1,float16,fp8,0,2.9713493982950845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,2,128,1,float16,fp8,0,2.0447999636332193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,2,128,1,float16,float16,0,1.416650613149007
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,4,128,1,float16,float16,0,1.457680066426595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,4,128,1,float16,fp8,0,1.5017493565877278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,8,128,1,float16,float16,0,1.4688533147176106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,48,8,128,1,float16,fp8,0,1.484458605448405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,2,128,1,float16,float16,0,13.740768432617188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,2,128,1,float16,fp8,0,12.723082224527994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,1,128,1,float16,float16,0,14.330132802327475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,4,128,1,float16,float16,0,14.646191914876303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,1,128,1,float16,fp8,0,21.181658426920574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,4,128,1,float16,fp8,0,13.314175923665365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,8,128,1,float16,float16,0,14.720458984375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,1,128,1,float16,float16,0,6.712805430094401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,48,128,1,float16,float16,0,7.6275679270426435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,48,128,1,float16,fp8,0,7.249765396118164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,48,8,128,1,float16,fp8,0,13.216288248697916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,1,128,1,float16,fp8,0,6.292144139607747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,2,128,1,float16,fp8,0,6.3173065185546875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,4,128,1,float16,float16,0,6.9968109130859375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,8,128,1,float16,float16,0,7.0415999094645185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,4,128,1,float16,fp8,0,7.158287684122722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,48,128,1,float16,float16,0,3.7043307622273765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,48,128,1,float16,fp8,0,3.6715094248453775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,8,128,1,float16,fp8,0,6.581845601399739
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,1,128,1,float16,float16,0,3.250682512919108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,1,128,1,float16,fp8,0,3.4675572713216147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,2,128,1,float16,float16,0,3.2772852579752603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,48,2,128,1,float16,float16,0,15.539957682291666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,2,128,1,float16,fp8,0,3.208597183227539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,4,128,1,float16,fp8,0,3.3366400400797525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,4,128,1,float16,float16,0,3.342405319213867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,48,128,1,float16,float16,0,1.86135466893514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,8,128,1,float16,float16,0,3.3439626693725586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,48,8,128,1,float16,fp8,0,3.612250645955404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,48,128,1,float16,fp8,0,2.016581376393636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,1,128,1,float16,fp8,0,1.6152480443318684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,2,128,1,float16,float16,0,1.618399937947591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,1,128,1,float16,float16,0,2.8219146728515625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,4,128,1,float16,float16,0,1.6707359949747722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,2,128,1,float16,fp8,0,1.762272040049235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,4,128,1,float16,fp8,0,1.7991414070129395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,8,128,1,float16,float16,0,1.692954699198405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,48,8,128,1,float16,fp8,0,1.7811412811279297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,48,128,1,float16,fp8,0,0.9778560002644857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,48,128,1,float16,float16,0,0.9506933689117432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,1,128,1,float16,float16,0,1.142133315404256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,1,128,1,float16,fp8,0,0.857215960820516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,2,128,1,float16,float16,0,0.8463946978251139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,2,128,1,float16,fp8,0,0.8803040186564127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,4,128,1,float16,float16,0,0.8663360277811686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,4,128,1,float16,fp8,0,0.9036479791005453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,8,128,1,float16,float16,0,0.8670879999796549
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,48,8,128,1,float16,fp8,0,0.8903146584828695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,1,128,1,float16,fp8,0,11.86404800415039
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,2,128,1,float16,fp8,0,11.99826176961263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,2,128,1,float16,float16,0,13.526959737141928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,4,128,1,float16,float16,0,13.654501597086588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,4,128,1,float16,fp8,0,12.813397725423178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,8,128,1,float16,float16,0,13.819082895914713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,8,128,1,float16,fp8,0,12.834964752197266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,1,128,1,float16,float16,0,6.20201047261556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,1,128,1,float16,fp8,0,5.965477625528972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,2,128,1,float16,float16,0,6.427482604980469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,48,128,1,float16,float16,0,7.458394368489583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,48,128,1,float16,fp8,0,7.231599807739258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,2,128,1,float16,fp8,0,6.0359039306640625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,4,128,1,float16,float16,0,6.7424055735270185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,48,1,128,1,float16,float16,0,30.650192260742188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,48,128,1,float16,float16,0,3.612943967183431
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,1,128,1,float16,float16,0,2.9253174463907876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,48,128,1,float16,fp8,0,3.643418629964193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,1,128,1,float16,fp8,0,3.3041919072469077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,4,128,1,float16,fp8,0,6.745983759562175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,8,128,1,float16,float16,0,6.412352244059245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,48,8,128,1,float16,fp8,0,6.769909540812175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,48,128,1,float16,float16,0,1.8275893529256184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,2,128,1,float16,fp8,0,3.187045415242513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,48,128,1,float16,fp8,0,1.8706293106079102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,2,128,1,float16,float16,0,8.929306666056315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,4,128,1,float16,float16,0,3.150357246398926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,8,128,1,float16,float16,0,3.195098559061686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,4,128,1,float16,fp8,0,3.2038294474283853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,48,8,128,1,float16,fp8,0,3.2694454193115234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,2,128,1,float16,float16,0,1.5041866302490234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,1,128,1,float16,fp8,0,1.5534987449645996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,1,128,1,float16,float16,0,1.4937280019124348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,2,128,1,float16,fp8,0,2.7617012659708657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,4,128,1,float16,float16,0,1.585792064666748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,4,128,1,float16,fp8,0,1.6758507092793782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,8,128,1,float16,float16,0,1.599199930826823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,48,128,1,float16,float16,0,0.918837308883667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,48,8,128,1,float16,fp8,0,1.6597867012023926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,48,128,1,float16,fp8,0,0.9699040253957113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,1,128,1,float16,float16,0,0.768501361211141
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,1,128,1,float16,fp8,0,0.8141600290934244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,2,128,1,float16,float16,0,0.7762773036956787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,2,128,1,float16,fp8,0,0.8065493106842041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,4,128,1,float16,float16,0,0.8078400293986002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,4,128,1,float16,fp8,0,0.8448106447855631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,48,128,1,float16,float16,0,0.48535998662312824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,8,128,1,float16,float16,0,0.8194826443990072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,48,8,128,1,float16,fp8,0,0.875210682551066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,2,128,1,float16,float16,0,0.4022293488184611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,48,128,1,float16,fp8,0,0.518943985303243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,1,128,1,float16,float16,0,0.39657068252563477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,1,128,1,float16,fp8,0,0.42366933822631836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,4,128,1,float16,float16,0,0.41924798488616943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,2,128,1,float16,fp8,0,0.4320106506347656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,8,128,1,float16,float16,0,0.426474650700887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,8,128,1,float16,fp8,0,0.4628640015920003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,48,4,128,1,float16,fp8,0,0.45287466049194336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,2,128,1,float16,float16,0,7.0336958567301435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,2,128,1,float16,fp8,0,7.369050979614258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,4,128,1,float16,fp8,0,7.829061508178711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,8,128,1,float16,float16,0,7.6631520589192705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,4,128,1,float16,float16,0,7.983290354410808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,1,128,1,float16,float16,0,7.204373041788737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,1,128,1,float16,float16,0,3.5922187169392905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,48,128,1,float16,float16,0,4.542336146036784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,48,128,1,float16,fp8,0,4.602842648824056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,1,128,1,float16,fp8,0,7.172159830729167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,1,128,1,float16,fp8,0,3.7517547607421875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,2,128,1,float16,float16,0,3.5797332127889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,48,128,1,float16,float16,0,2.2466932932535806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,4,128,1,float16,fp8,0,3.9421653747558594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,8,128,1,float16,float16,0,3.821221351623535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,48,128,1,float16,fp8,0,2.410517374674479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,2,128,1,float16,fp8,0,3.632517178853353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,4,128,1,float16,float16,0,3.795498530069987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,48,8,128,1,float16,fp8,0,3.950410525004069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,1,128,1,float16,float16,0,1.7586347262064617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,2,128,1,float16,float16,0,1.8252159754435222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,1,128,1,float16,fp8,0,1.806879997253418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,2,128,1,float16,fp8,0,1.8435786565144856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,4,128,1,float16,float16,0,1.8944212595621746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,4,128,1,float16,fp8,0,2.0926987330118814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,8,128,1,float16,float16,0,1.9074187278747559
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,48,128,1,float16,float16,0,1.1384373505910237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,48,8,128,1,float16,fp8,0,2.046346664428711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,48,8,128,1,float16,fp8,0,17.705567677815754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,48,128,1,float16,fp8,0,1.187824010848999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,1,128,1,float16,float16,0,0.8844426472981771
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,2,128,1,float16,fp8,0,0.9456373055775961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,1,128,1,float16,fp8,0,0.9476586977640787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,2,128,1,float16,float16,0,0.9217120011647543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,4,128,1,float16,float16,0,0.971776008605957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,48,128,1,float16,float16,0,0.5860480070114136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,4,128,1,float16,fp8,0,1.041759967803955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,8,128,1,float16,fp8,0,1.0261653264363606
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,48,8,128,1,float16,float16,0,1.4732960065205891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,48,128,1,float16,fp8,0,0.6239893436431885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,1,128,1,float16,float16,0,0.45657598972320557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,1,128,1,float16,fp8,0,0.4925493399302165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,2,128,1,float16,float16,0,0.47174398104349774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,8,128,1,float16,fp8,0,0.5414133469263712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,2,128,1,float16,fp8,0,0.5005760192871094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,4,128,1,float16,float16,0,0.4949066638946533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,4,128,1,float16,fp8,0,0.5288266738255819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,48,8,128,1,float16,float16,0,0.5455466508865356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,48,128,1,float16,float16,0,0.3123040000597636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,48,128,1,float16,fp8,0,0.3394826650619507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,1,128,1,float16,float16,0,0.2451253334681193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,1,128,1,float16,fp8,0,0.26027733087539673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,2,128,1,float16,float16,0,0.25310399134953815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,2,128,1,float16,fp8,0,0.2672800024350484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,4,128,1,float16,float16,0,0.26233067115147907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,4,128,1,float16,fp8,0,0.2922240098317464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,8,128,1,float16,float16,0,0.27214399973551434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,48,8,128,1,float16,fp8,0,0.2964373429616292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,1,128,1,float16,float16,0,6.855653127034505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,1,128,1,float16,fp8,0,7.140192031860352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,4,128,1,float16,float16,0,7.731168111165364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,2,128,1,float16,fp8,0,7.212912241617839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,2,128,1,float16,float16,0,7.648042678833008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,8,128,1,float16,float16,0,7.934853235880534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,8,128,1,float16,fp8,0,7.975525538126628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,48,128,1,float16,float16,0,4.677344004313151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,48,128,1,float16,fp8,0,4.904800097147624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,1,128,1,float16,float16,0,3.707167943318685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,1,128,1,float16,fp8,0,3.5435307820638022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,2,128,1,float16,float16,0,3.5044692357381186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,2,128,1,float16,fp8,0,3.774143854777018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,48,4,128,1,float16,fp8,0,13.969322204589844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,4,128,1,float16,float16,0,3.807999928792318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,48,128,1,float16,float16,0,2.34004799524943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,4,128,1,float16,fp8,0,4.18939208984375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,1,128,1,float16,float16,0,1.689674695332845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,8,128,1,float16,float16,0,3.7766186396280923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,2,128,1,float16,float16,0,1.8124693234761555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,48,128,1,float16,fp8,0,2.5572053591410318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,48,8,128,1,float16,fp8,0,4.141754786173503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,1,128,1,float16,fp8,0,2.5844319661458335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,2,128,1,float16,fp8,0,1.881813367207845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,4,128,1,float16,float16,0,1.9096906979878743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,4,128,1,float16,fp8,0,2.0730080604553223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,8,128,1,float16,float16,0,1.9514133135477703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,48,128,1,float16,float16,0,1.1805866559346516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,48,8,128,1,float16,fp8,0,2.0922239621480307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,48,128,1,float16,fp8,0,1.4260586102803547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,1,128,1,float16,float16,0,0.8642400105794271
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,1,128,1,float16,fp8,0,0.9311040242513021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,2,128,1,float16,float16,0,0.8793439865112305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,2,128,1,float16,fp8,0,0.958079973856608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,4,128,1,float16,float16,0,0.9613653024037679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,4,128,1,float16,fp8,0,1.0360586643218994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,8,128,1,float16,fp8,0,1.0700960159301758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,48,8,128,1,float16,float16,0,1.3526612917582195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,48,128,1,float16,float16,0,0.6043146848678589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,48,128,1,float16,fp8,0,0.6453386545181274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,1,128,1,float16,float16,0,0.44422932465871173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,1,128,1,float16,fp8,0,0.47497065862019855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,2,128,1,float16,float16,0,0.45523734887441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,4,128,1,float16,fp8,0,0.5446346600850424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,2,128,1,float16,fp8,0,0.49380799134572345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,4,128,1,float16,float16,0,0.5975893338521322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,8,128,1,float16,float16,0,0.5018239816029867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,48,8,128,1,float16,fp8,0,0.5492053429285685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,48,128,1,float16,float16,0,0.32132800420125324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,48,128,1,float16,fp8,0,0.34592533111572266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,1,128,1,float16,float16,0,0.23735467592875162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,4,128,1,float16,float16,0,0.2632533311843872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,1,128,1,float16,fp8,0,0.2598186731338501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,2,128,1,float16,float16,0,0.24528000752131143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,4,128,1,float16,fp8,0,0.2879040042559306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,48,128,1,float16,fp8,0,0.19524266322453818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,2,128,1,float16,fp8,0,0.26427199443181354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,8,128,1,float16,fp8,0,0.2893173297246297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,48,8,128,1,float16,float16,0,0.27110399802525836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,48,128,1,float16,float16,0,0.17697066068649292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,1,128,1,float16,float16,0,0.13124799728393555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,4,128,1,float16,fp8,0,0.15811199943224588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,1,128,1,float16,fp8,0,0.1389173368612925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,2,128,1,float16,float16,0,0.13524799545605978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,2,128,1,float16,fp8,0,0.14519466956456503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,4,128,1,float16,float16,0,0.14592533310254416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,8,128,1,float16,float16,0,0.15038399895032248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,48,8,128,1,float16,fp8,0,0.16412267088890076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,1,128,1,float16,float16,0,4.212426821390788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,4,128,1,float16,float16,0,4.801610628763835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,4,128,1,float16,fp8,0,5.242885271708171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,8,128,1,float16,float16,0,4.768213272094727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,1,128,1,float16,fp8,0,4.41816520690918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,48,128,1,float16,float16,0,3.0338080724080405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,8,128,1,float16,fp8,0,5.226570765177409
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,1,128,1,float16,float16,0,2.0416372617085776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,2,128,1,float16,float16,0,8.064314524332682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,48,128,1,float16,fp8,0,3.323754628499349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,1,128,1,float16,fp8,0,2.2676053047180176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,2,128,1,float16,float16,0,2.144848028818766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,48,2,128,1,float16,fp8,0,4.565605481465657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,4,128,1,float16,float16,0,2.360864003499349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,2,128,1,float16,fp8,0,2.335434595743815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,4,128,1,float16,fp8,0,2.589066664377848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,48,128,1,float16,float16,0,1.5191839536031086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,8,128,1,float16,fp8,0,2.6483786900838218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,48,128,1,float16,fp8,0,1.6654987335205078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,1,128,1,float16,float16,0,1.034826676050822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,1,128,1,float16,fp8,0,1.1500319639841716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,2,128,1,float16,float16,0,1.1127999623616536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,48,8,128,1,float16,float16,0,4.879359881083171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,2,128,1,float16,fp8,0,1.1595733165740967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,4,128,1,float16,float16,0,1.2061546643575032
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,4,128,1,float16,fp8,0,1.3345333735148113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,8,128,1,float16,float16,0,1.2098506291707356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,48,128,1,float16,float16,0,0.77292799949646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,48,8,128,1,float16,fp8,0,1.3185919920603435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,48,128,1,float16,fp8,0,0.8630080223083496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,1,128,1,float16,fp8,0,0.579589327176412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,1,128,1,float16,float16,0,0.6161973476409912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,2,128,1,float16,fp8,0,0.5966026782989502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,2,128,1,float16,float16,0,0.5694346825281779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,4,128,1,float16,float16,0,0.6137866576512655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,4,128,1,float16,fp8,0,0.686293363571167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,8,128,1,float16,float16,0,0.6149119933446249
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,48,8,128,1,float16,fp8,0,0.6928213437398275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,48,128,1,float16,float16,0,0.39688531557718915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,48,128,1,float16,fp8,0,0.44011199474334717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,1,128,1,float16,float16,0,0.2786346673965454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,1,128,1,float16,fp8,0,0.3107946713765462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,2,128,1,float16,float16,0,0.29598400990168255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,8,128,1,float16,fp8,0,0.3568640152613322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,48,128,1,float16,float16,0,0.21665600935618082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,48,128,1,float16,fp8,0,0.23907200495402017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,2,128,1,float16,fp8,0,0.3206560015678406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,2,128,1,float16,float16,0,0.16008533040682474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,4,128,1,float16,float16,0,0.3227573235829671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,4,128,1,float16,fp8,0,0.35438398520151776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,48,8,128,1,float16,float16,0,0.32953067620595294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,1,128,1,float16,float16,0,0.15332266688346863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,1,128,1,float16,fp8,0,0.16902399063110352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,2,128,1,float16,fp8,0,0.17407466967900595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,4,128,1,float16,float16,0,0.1771519978841146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,4,128,1,float16,fp8,0,0.19022399187088013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,8,128,1,float16,float16,0,0.17638933658599854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,48,8,128,1,float16,fp8,0,0.19772799809773764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,48,128,1,float16,float16,0,0.11987200379371643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,48,128,1,float16,fp8,0,0.13472533226013184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,1,128,1,float16,float16,0,0.08679466446240743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,1,128,1,float16,fp8,0,0.09577600161234538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,2,128,1,float16,float16,0,0.08861333131790161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,2,128,1,float16,fp8,0,0.09924800197283427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,4,128,1,float16,float16,0,0.0979146659374237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,4,128,1,float16,fp8,0,0.10660266876220703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,8,128,1,float16,float16,0,0.09945066769917806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,48,8,128,1,float16,fp8,0,0.11075733105341594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,1,128,1,float16,fp8,0,4.7527360916137695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,2,128,1,float16,fp8,0,4.875034650166829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,2,128,1,float16,float16,0,4.570501327514648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,8,128,1,float16,float16,0,5.069631894429524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,4,128,1,float16,float16,0,5.086880048116048
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,1,128,1,float16,float16,0,4.277599970499675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,48,128,1,float16,float16,0,3.390746752421061
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,4,128,1,float16,fp8,0,8.054789225260416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,1,128,1,float16,fp8,0,2.407269318898519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,1,128,1,float16,float16,0,2.12880007425944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,48,128,1,float16,fp8,0,3.8095572789510093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,2,128,1,float16,float16,0,2.2316266695658364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,2,128,1,float16,fp8,0,2.4989120165506997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,48,128,1,float16,float16,0,1.6914292971293132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,4,128,1,float16,float16,0,2.553349335988363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,8,128,1,float16,float16,0,2.589914639790853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,8,128,1,float16,fp8,0,2.8427041371663413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,1,128,1,float16,float16,0,1.0796213150024414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,48,128,1,float16,fp8,0,1.9219519297281902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,1,128,1,float16,fp8,0,1.1986773014068604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,48,4,128,1,float16,fp8,0,4.033962567647298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,2,128,1,float16,fp8,0,1.2431999842325847
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,2,128,1,float16,float16,0,1.140928030014038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,4,128,1,float16,float16,0,1.282047986984253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,8,128,1,float16,float16,0,1.3034453392028809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,4,128,1,float16,fp8,0,1.4416054089864094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,48,8,128,1,float16,fp8,0,1.4838666915893555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,48,128,1,float16,float16,0,0.9708693027496338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,1,128,1,float16,float16,0,0.5643839836120605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,48,8,128,1,float16,fp8,0,5.6987253824869795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,48,128,1,float16,fp8,0,0.9801066716512045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,2,128,1,float16,float16,0,0.5875893433888754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,1,128,1,float16,fp8,0,0.6115466753641764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,2,128,1,float16,fp8,0,0.6374773184458414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,4,128,1,float16,float16,0,0.664624015490214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,8,128,1,float16,float16,0,0.6806506315867106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,4,128,1,float16,fp8,0,0.8004106680552164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,48,128,1,float16,float16,0,0.444976011912028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,48,128,1,float16,fp8,0,0.5043466488520304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,1,128,1,float16,float16,0,0.2875093420346578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,1,128,1,float16,fp8,0,0.3215786616007487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,48,8,128,1,float16,fp8,0,0.7526880105336508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,2,128,1,float16,float16,0,0.30669333537419635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,2,128,1,float16,fp8,0,0.33529067039489746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,48,128,1,float16,float16,0,0.24310932556788126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,4,128,1,float16,float16,0,0.34810133775075275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,8,128,1,float16,float16,0,0.34619200229644775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,4,128,1,float16,fp8,0,0.38254400094350177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,48,8,128,1,float16,fp8,0,0.3903626600901286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,48,128,1,float16,fp8,0,0.2756426731745402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,1,128,1,float16,float16,0,0.15638933579126993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,1,128,1,float16,fp8,0,0.17491199572881064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,8,128,1,float16,float16,0,0.1883466641108195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,2,128,1,float16,float16,0,0.16371200482050577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,2,128,1,float16,fp8,0,0.18029866615931192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,4,128,1,float16,float16,0,0.185754656791687
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,4,128,1,float16,fp8,0,0.20495466391245523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,48,8,128,1,float16,fp8,0,0.208079993724823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,48,128,1,float16,float16,0,0.12979732950528464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,48,128,1,float16,fp8,0,0.14839466412862143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,1,128,1,float16,float16,0,0.08940266569455464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,8,128,1,float16,float16,0,0.10388799508412679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,1,128,1,float16,fp8,0,0.10033067067464192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,2,128,1,float16,float16,0,0.09119466940561931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,2,128,1,float16,fp8,0,0.1039626697699229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,4,128,1,float16,float16,0,0.1020906666914622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,4,128,1,float16,fp8,0,0.11481600006421407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,48,8,128,1,float16,fp8,0,0.11733333269755046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,48,128,1,float16,float16,0,0.07530133426189423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,4,128,1,float16,float16,0,0.0547680010398229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,48,128,1,float16,fp8,0,0.08889066179593404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,1,128,1,float16,float16,0,0.046629334489504494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,1,128,1,float16,fp8,0,0.05394133428732554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,2,128,1,float16,float16,0,0.05161066850026449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,2,128,1,float16,fp8,0,0.057333335280418396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,4,128,1,float16,fp8,0,0.06237866481145223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,8,128,1,float16,float16,0,0.05942933261394501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,48,8,128,1,float16,fp8,0,0.06317866841952006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,1,128,1,float16,float16,0,3.1456851959228516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,2,128,1,float16,float16,0,5.4443308512369795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,48,128,1,float16,float16,0,2.8110132217407227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,1,128,1,float16,fp8,0,3.542319933573405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,4,128,1,float16,float16,0,3.9580373764038086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,2,128,1,float16,fp8,0,3.7599732081095376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,4,128,1,float16,fp8,0,4.5607147216796875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,1,128,1,float16,float16,0,1.5790133476257324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,48,128,1,float16,fp8,0,3.969327926635742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,8,128,1,float16,float16,0,4.017978668212891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,48,8,128,1,float16,fp8,0,4.650437355041504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,2,128,1,float16,float16,0,1.7085760434468586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,1,128,1,float16,fp8,0,1.777136007944743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,2,128,1,float16,fp8,0,1.8802453676859539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,48,128,1,float16,float16,0,1.4166879653930664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,4,128,1,float16,fp8,0,2.3145813941955566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,4,128,1,float16,float16,0,1.9997386932373047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,1,128,1,float16,float16,0,0.7986186345418295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,8,128,1,float16,float16,0,1.988320032755534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,48,128,1,float16,fp8,0,1.7069120407104492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,48,8,128,1,float16,fp8,0,2.3112266858418784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,1,128,1,float16,fp8,0,0.9011253515879313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,4,128,1,float16,float16,0,1.0026933352152507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,48,128,1,float16,float16,0,0.7300373713175455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,2,128,1,float16,float16,0,0.8784586588541666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,8,128,1,float16,fp8,0,1.2003680070241292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,8,128,1,float16,float16,0,1.0288480122884114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,1,128,1,float16,float16,0,0.41094398498535156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,2,128,1,float16,fp8,0,0.9530080159505209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,48,128,1,float16,fp8,0,0.8655200004577637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,48,4,128,1,float16,fp8,0,1.1616426308949788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,1,128,1,float16,fp8,0,0.46350399653116864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,2,128,1,float16,float16,0,0.4517973264058431
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,8,128,1,float16,float16,0,0.5219573179880778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,2,128,1,float16,fp8,0,0.4928373495737712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,48,128,1,float16,float16,0,0.3715573151906331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,4,128,1,float16,float16,0,0.5137439966201782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,48,128,1,float16,fp8,0,0.4487786690394084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,4,128,1,float16,fp8,0,0.5944106578826904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,48,8,128,1,float16,fp8,0,0.6150613228480021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,1,128,1,float16,float16,0,0.23026132583618164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,1,128,1,float16,fp8,0,0.2446720004081726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,2,128,1,float16,float16,0,0.23409066597620645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,2,128,1,float16,fp8,0,0.25967999299367267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,4,128,1,float16,float16,0,0.267194668451945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,4,128,1,float16,fp8,0,0.30966933568318683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,8,128,1,float16,float16,0,0.2733866572380066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,48,8,128,1,float16,fp8,0,0.319813330968221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,48,128,1,float16,float16,0,0.19961599508921304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,48,128,1,float16,fp8,0,0.23810666799545288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,1,128,1,float16,float16,0,0.11900800466537476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,1,128,1,float16,fp8,0,0.13447999954223633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,2,128,1,float16,float16,0,0.12678933143615723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,2,128,1,float16,fp8,0,0.1418186624844869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,4,128,1,float16,float16,0,0.1427786648273468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,4,128,1,float16,fp8,0,0.16406400005022684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,8,128,1,float16,float16,0,0.14722667137781778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,48,8,128,1,float16,fp8,0,0.17194666465123495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,48,128,1,float16,float16,0,0.10822400450706482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,48,128,1,float16,fp8,0,0.13303466637929282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,1,128,1,float16,float16,0,0.06921599805355072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,1,128,1,float16,fp8,0,0.07780266801516215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,2,128,1,float16,float16,0,0.07140799860159557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,2,128,1,float16,fp8,0,0.08284266789754231
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,4,128,1,float16,float16,0,0.07967466612656911
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,4,128,1,float16,fp8,0,0.09291733304659526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,8,128,1,float16,float16,0,0.08110400040944417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,48,8,128,1,float16,fp8,0,0.09726933638254802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,2,128,1,float16,fp8,0,0.045184001326560974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,48,128,1,float16,float16,0,0.06324266890684764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,48,128,1,float16,fp8,0,0.07698666552702586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,1,128,1,float16,float16,0,0.03595733394225439
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,1,128,1,float16,fp8,0,0.042677332957585655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,2,128,1,float16,float16,0,0.040207999447981514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,4,128,1,float16,float16,0,0.04227200150489807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,4,128,1,float16,fp8,0,0.0492799977461497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,8,128,1,float16,float16,0,0.0443200021982193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,48,8,128,1,float16,fp8,0,0.05070933202902476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,48,128,1,float16,float16,0,0.03601066768169403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,4,128,1,float16,float16,0,0.027461332579453785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,48,128,1,float16,fp8,0,0.04292266567548116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,1,128,1,float16,float16,0,0.026586666703224182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,1,128,1,float16,fp8,0,0.029904000461101532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,2,128,1,float16,float16,0,0.026399999856948853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,2,128,1,float16,fp8,0,0.03012799968322118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,4,128,1,float16,fp8,0,0.03331200033426285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,8,128,1,float16,float16,0,0.027621333797772724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,48,8,128,1,float16,fp8,0,0.03475199888149897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,1,128,1,float16,float16,0,1.313418706258138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,1,128,1,float16,fp8,0,1.478256066640218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,2,128,1,float16,fp8,0,1.613642692565918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,4,128,1,float16,float16,0,1.7245866457621257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,48,128,1,float16,fp8,0,1.4769280751546223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,4,128,1,float16,fp8,0,2.022085348765055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,48,128,1,float16,float16,0,1.297983964284261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,2,128,1,float16,float16,0,1.41811736424764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,1,128,1,float16,fp8,0,0.7518880367279053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,1,128,1,float16,float16,0,0.6663039922714233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,8,128,1,float16,float16,0,1.7380320231119792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,48,8,128,1,float16,fp8,0,2.0445653597513833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,2,128,1,float16,float16,0,0.7285493214925131
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,2,128,1,float16,fp8,0,0.8183680375417074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,4,128,1,float16,float16,0,0.8764106432596842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,4,128,1,float16,fp8,0,1.0207786560058594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,8,128,1,float16,float16,0,0.8876266479492188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,48,8,128,1,float16,fp8,0,1.0670026938120525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,1,128,1,float16,float16,0,0.34279465675354004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,48,128,1,float16,fp8,0,0.7544586658477783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,48,128,1,float16,float16,0,0.6602986653645834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,1,128,1,float16,fp8,0,0.38842666149139404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,2,128,1,float16,float16,0,0.3824426730473836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,2,128,1,float16,fp8,0,0.4216373364130656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,4,128,1,float16,float16,0,0.4474506775538127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,4,128,1,float16,fp8,0,0.5216853221257528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,8,128,1,float16,float16,0,0.4566719929377238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,48,8,128,1,float16,fp8,0,0.5476426680882772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,48,128,1,float16,float16,0,0.3407466808954875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,48,128,1,float16,fp8,0,0.39363733927408856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,1,128,1,float16,float16,0,0.18221867084503174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,1,128,1,float16,fp8,0,0.20726933081944784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,2,128,1,float16,float16,0,0.19930134216944376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,2,128,1,float16,fp8,0,0.22236265738805136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,4,128,1,float16,float16,0,0.23268800973892212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,4,128,1,float16,fp8,0,0.27332266171773273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,8,128,1,float16,float16,0,0.23809599876403809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,48,8,128,1,float16,fp8,0,0.28480533758799237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,48,128,1,float16,float16,0,0.1814240018526713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,48,128,1,float16,fp8,0,0.20871466398239136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,1,128,1,float16,float16,0,0.1018453339735667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,1,128,1,float16,fp8,0,0.11496532956759135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,2,128,1,float16,float16,0,0.10732799768447876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,2,128,1,float16,fp8,0,0.12357866764068604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,4,128,1,float16,float16,0,0.12427199880282085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,4,128,1,float16,fp8,0,0.14619732896486917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,8,128,1,float16,float16,0,0.1296266714731852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,48,8,128,1,float16,fp8,0,0.15268799662590027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,48,128,1,float16,float16,0,0.1009279986222585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,48,128,1,float16,fp8,0,0.11588799953460693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,1,128,1,float16,float16,0,0.057999998331069946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,1,128,1,float16,fp8,0,0.06765866776307423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,2,128,1,float16,float16,0,0.06118933359781901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,2,128,1,float16,fp8,0,0.07117333511511485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,4,128,1,float16,float16,0,0.07090133428573608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,4,128,1,float16,fp8,0,0.0830026666323344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,1,128,1,float16,fp8,0,0.03597866743803024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,8,128,1,float16,float16,0,0.07209066549936931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,2,128,1,float16,fp8,0,0.03832533210515976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,48,8,128,1,float16,fp8,0,0.08659733335177104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,48,128,1,float16,float16,0,0.057146668434143066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,4,128,1,float16,fp8,0,0.04194133480389913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,48,128,1,float16,fp8,0,0.06761600077152252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,1,128,1,float16,float16,0,0.030693332354227703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,2,128,1,float16,float16,0,0.032218667368094124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,4,128,1,float16,float16,0,0.035461333890755974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,8,128,1,float16,float16,0,0.03885333240032196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,48,8,128,1,float16,fp8,0,0.04448533554871877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,48,128,1,float16,float16,0,0.03270400067170461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,48,128,1,float16,fp8,0,0.039061332742373146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,4,128,1,float16,fp8,0,0.027749332288901012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,1,128,1,float16,float16,0,0.0229120006163915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,1,128,1,float16,fp8,0,0.025818665822347004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,2,128,1,float16,float16,0,0.02239999920129776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,2,128,1,float16,fp8,0,0.0259253333012263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,1,128,1,float16,float16,0,0.01623999948302905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,4,128,1,float16,float16,0,0.023621333142121632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,8,128,1,float16,float16,0,0.023183998962243397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,48,8,128,1,float16,fp8,0,0.02863466739654541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,48,128,1,float16,float16,0,0.018474667022625606
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,48,128,1,float16,fp8,0,0.02310933421055476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,1,128,1,float16,fp8,0,0.019808000574509304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,2,128,1,float16,float16,0,0.01676799977819125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,2,128,1,float16,fp8,0,0.019653332730134327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,4,128,1,float16,float16,0,0.017279999951521557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,4,128,1,float16,fp8,0,0.019567999988794327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,8,128,1,float16,float16,0,0.01703466723362605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,48,8,128,1,float16,fp8,0,0.020762667059898376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,1,128,1,float16,float16,0,0.6375573476155599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,1,128,1,float16,fp8,0,0.6853973070780436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,2,128,1,float16,float16,0,0.6973439852396647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,2,128,1,float16,fp8,0,0.7321866353352865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,4,128,1,float16,float16,0,0.842192014058431
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,48,128,1,float16,float16,0,0.63209601243337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,1,128,1,float16,float16,0,0.3277440071105957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,48,128,1,float16,fp8,0,0.6857866446177164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,1,128,1,float16,fp8,0,0.35630400975545246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,2,128,1,float16,float16,0,0.3664906819661458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,8,128,1,float16,float16,0,0.8621066411336263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,4,128,1,float16,fp8,0,0.9441760381062826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,2,128,1,float16,fp8,0,0.38067201773325604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,4,128,1,float16,float16,0,0.4285600185394287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,4,128,1,float16,fp8,0,0.4861066738764445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,8,128,1,float16,float16,0,0.44069333871205646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,48,8,128,1,float16,fp8,0,0.4965706666310628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,48,128,1,float16,float16,0,0.32715733846028644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,1,128,1,float16,float16,0,0.17360534270604452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,48,8,128,1,float16,fp8,0,0.9773866335550944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,2,128,1,float16,fp8,0,0.20428800582885742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,48,128,1,float16,fp8,0,0.3527573347091675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,1,128,1,float16,fp8,0,0.19117865959803262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,2,128,1,float16,float16,0,0.19047999382019043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,4,128,1,float16,float16,0,0.22293333212534586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,4,128,1,float16,fp8,0,0.2500266631444295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,48,128,1,float16,float16,0,0.1743519902229309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,8,128,1,float16,fp8,0,0.26450133323669434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,48,8,128,1,float16,float16,0,0.22929600874582926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,4,128,1,float16,float16,0,0.11873066425323486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,48,128,1,float16,fp8,0,0.19024000565210977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,1,128,1,float16,float16,0,0.09473599990208943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,4,128,1,float16,fp8,0,0.1388320028781891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,1,128,1,float16,fp8,0,0.1092746655146281
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,8,128,1,float16,fp8,0,0.14144532879193625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,2,128,1,float16,float16,0,0.10334933797518413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,2,128,1,float16,fp8,0,0.11478933691978455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,48,8,128,1,float16,float16,0,0.12124266227086385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,48,128,1,float16,float16,0,0.09643733501434326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,48,128,1,float16,fp8,0,0.1046560009320577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,1,128,1,float16,float16,0,0.05508799850940704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,1,128,1,float16,fp8,0,0.06150400141874949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,2,128,1,float16,float16,0,0.05877333382765452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,2,128,1,float16,fp8,0,0.06486933430035909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,4,128,1,float16,float16,0,0.06676800052324931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,8,128,1,float16,float16,0,0.07004266480604808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,8,128,1,float16,fp8,0,0.0790293316046397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,48,4,128,1,float16,fp8,0,0.0777706652879715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,48,128,1,float16,float16,0,0.054976001381874084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,48,128,1,float16,fp8,0,0.05899733304977417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,1,128,1,float16,float16,0,0.02773333340883255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,1,128,1,float16,fp8,0,0.030997333427270252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,2,128,1,float16,float16,0,0.02939733366171519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,2,128,1,float16,fp8,0,0.03278400003910065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,4,128,1,float16,float16,0,0.03238933285077413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,8,128,1,float16,float16,0,0.033471999069054924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,4,128,1,float16,fp8,0,0.036415999134381614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,48,8,128,1,float16,fp8,0,0.03711999952793121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,48,128,1,float16,float16,0,0.032186667124430336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,48,128,1,float16,fp8,0,0.031557333966096245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,1,128,1,float16,float16,0,0.021375998854637146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,1,128,1,float16,fp8,0,0.02384000023206075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,8,128,1,float16,float16,0,0.02292799949645996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,8,128,1,float16,fp8,0,0.025605333348115284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,48,128,1,float16,fp8,0,0.019760000209013622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,2,128,1,float16,float16,0,0.021125334004561108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,2,128,1,float16,fp8,0,0.0239680012067159
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,4,128,1,float16,float16,0,0.021701333423455555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,48,4,128,1,float16,fp8,0,0.02555199960867564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,48,128,1,float16,float16,0,0.017263999829689663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,1,128,1,float16,float16,0,0.015520000209410986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,1,128,1,float16,fp8,0,0.017866666118303936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,2,128,1,float16,float16,0,0.015402667224407196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,2,128,1,float16,fp8,0,0.01762666677435239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,48,128,1,float16,float16,0,0.015813333292802174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,48,128,1,float16,fp8,0,0.0183146670460701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,4,128,1,float16,float16,0,0.015013333410024643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,4,128,1,float16,fp8,0,0.017583999782800674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,2,128,1,float16,fp8,0,0.017269333203633625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,8,128,1,float16,float16,0,0.015423999478419622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,48,8,128,1,float16,fp8,0,0.017909333109855652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,8,128,1,float16,float16,0,0.015018666783968607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,8,128,1,float16,fp8,0,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,1,128,1,float16,float16,0,0.015146666516860327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,1,128,1,float16,fp8,0,0.017701332767804463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,2,128,1,float16,float16,0,0.01492799942692121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,4,128,1,float16,float16,0,0.014666666587193808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,48,4,128,1,float16,fp8,0,0.01747200017174085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,1,128,1,float16,fp8,0,0.3561333417892456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,2,128,1,float16,float16,0,0.36286401748657227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,4,128,1,float16,float16,0,0.43249066670735675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,1,128,1,float16,float16,0,0.32922667264938354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,8,128,1,float16,fp8,0,0.49956798553466797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,48,128,1,float16,float16,0,0.36378665765126544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,48,128,1,float16,fp8,0,0.359935998916626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,2,128,1,float16,fp8,0,0.3803306818008423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,8,128,1,float16,float16,0,0.43514664967854816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,1,128,1,float16,fp8,0,0.19381866852442423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,2,128,1,float16,float16,0,0.18890132506688437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,48,4,128,1,float16,fp8,0,0.4843306541442871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,1,128,1,float16,float16,0,0.17329599459966025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,8,128,1,float16,fp8,0,0.2638026674588521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,2,128,1,float16,fp8,0,0.20253866910934448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,48,128,1,float16,float16,0,0.19037334124247232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,48,128,1,float16,fp8,0,0.18941867351531982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,1,128,1,float16,float16,0,0.09531733393669128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,4,128,1,float16,fp8,0,0.25337066253026325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,8,128,1,float16,float16,0,0.22685867547988892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,48,4,128,1,float16,float16,0,0.22249066829681396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,1,128,1,float16,fp8,0,0.1093386709690094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,2,128,1,float16,float16,0,0.10264000296592712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,4,128,1,float16,fp8,0,0.13955733180046082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,2,128,1,float16,fp8,0,0.11603200435638428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,48,128,1,float16,fp8,0,0.1032960017522176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,1,128,1,float16,float16,0,0.05351466437180837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,4,128,1,float16,float16,0,0.11961066722869873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,8,128,1,float16,float16,0,0.12256532907485962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,2,128,1,float16,fp8,0,0.06597866614659627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,48,8,128,1,float16,fp8,0,0.14155200123786926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,48,128,1,float16,float16,0,0.10354133447011311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,1,128,1,float16,fp8,0,0.06150400141874949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,2,128,1,float16,float16,0,0.05821866790453593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,4,128,1,float16,float16,0,0.06682666639486949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,4,128,1,float16,fp8,0,0.07613333563009898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,8,128,1,float16,fp8,0,0.07978666822115581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,48,8,128,1,float16,float16,0,0.06962666908899943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,48,128,1,float16,float16,0,0.058933332562446594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,48,128,1,float16,fp8,0,0.055957332253456116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,1,128,1,float16,float16,0,0.027242665489514668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,1,128,1,float16,fp8,0,0.030991998811562855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,8,128,1,float16,float16,0,0.03363200028737386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,8,128,1,float16,fp8,0,0.03681066632270813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,2,128,1,float16,float16,0,0.028234665592511494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,2,128,1,float16,fp8,0,0.03339199970165888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,4,128,1,float16,float16,0,0.031471999982992806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,48,4,128,1,float16,fp8,0,0.03615466753641764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,48,128,1,float16,float16,0,0.03086400032043457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,48,128,1,float16,fp8,0,0.025983999172846477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,1,128,1,float16,float16,0,0.020842666427294414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,1,128,1,float16,fp8,0,0.02367466688156128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,2,128,1,float16,float16,0,0.020975999534130096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,2,128,1,float16,fp8,0,0.023749334116776783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,4,128,1,float16,float16,0,0.021935999393463135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,4,128,1,float16,fp8,0,0.025392000873883564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,8,128,1,float16,float16,0,0.02181333303451538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,1,128,1,float16,fp8,0,0.017850667238235474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,48,8,128,1,float16,fp8,0,0.025914666553338368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,48,128,1,float16,float16,0,0.017914666483799618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,4,128,1,float16,float16,0,0.015615999698638916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,48,128,1,float16,fp8,0,0.018613333503405254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,1,128,1,float16,float16,0,0.015290666371583939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,2,128,1,float16,float16,0,0.01524266724785169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,2,128,1,float16,fp8,0,0.017749333133300144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,4,128,1,float16,fp8,0,0.017866666118303936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,8,128,1,float16,float16,0,0.015082667271296183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,48,8,128,1,float16,fp8,0,0.017909333109855652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,48,128,1,float16,fp8,0,0.014458666245142618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,48,128,1,float16,float16,0,0.013722666849692663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,1,128,1,float16,float16,0,0.014741333822409311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,2,128,1,float16,float16,0,0.014666666587193808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,1,128,1,float16,fp8,0,0.017450666675964992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,2,128,1,float16,fp8,0,0.01752000053723653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,4,128,1,float16,float16,0,0.015034666905800501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,4,128,1,float16,fp8,0,0.017279999951521557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,8,128,1,float16,float16,0,0.01504533365368843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,2,128,1,float16,float16,0,0.014378666877746582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,48,8,128,1,float16,fp8,0,0.01709866647919019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,48,128,1,float16,float16,0,0.01332266628742218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,48,128,1,float16,fp8,0,0.014208000153303146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,1,128,1,float16,float16,0,0.014757333944241205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,1,128,1,float16,fp8,0,0.016810666769742966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,2,128,1,float16,fp8,0,0.016917333006858826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,4,128,1,float16,float16,0,0.014864000181357065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,4,128,1,float16,fp8,0,0.017045332739750545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,8,128,1,float16,float16,0,0.014933332800865173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,48,8,128,1,float16,fp8,0,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,1,128,1,float16,float16,0,0.1699733336766561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,2,128,1,float16,float16,0,0.18847999970118204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,2,128,1,float16,fp8,0,0.20313066244125366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,4,128,1,float16,float16,0,0.2227946718533834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,4,128,1,float16,fp8,0,0.254037340482076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,1,128,1,float16,fp8,0,0.19075200955073038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,48,128,1,float16,float16,0,0.26441067457199097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,8,128,1,float16,fp8,0,0.2975146571795146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,48,128,1,float16,fp8,0,0.2574186722437541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,1,128,1,float16,float16,0,0.09452266494433086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,48,8,128,1,float16,float16,0,0.26361600557963055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,1,128,1,float16,fp8,0,0.1090826690196991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,2,128,1,float16,float16,0,0.10108799735705058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,2,128,1,float16,fp8,0,0.11533332864443462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,4,128,1,float16,float16,0,0.11797866225242615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,8,128,1,float16,fp8,0,0.16185599565505981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,4,128,1,float16,fp8,0,0.13984533150990805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,48,128,1,float16,float16,0,0.13941333691279092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,48,8,128,1,float16,float16,0,0.13988799850145975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,48,128,1,float16,fp8,0,0.13570666313171387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,1,128,1,float16,float16,0,0.0539626677831014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,1,128,1,float16,fp8,0,0.06244266529877981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,2,128,1,float16,float16,0,0.05924266576766968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,8,128,1,float16,float16,0,0.07944533228874207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,2,128,1,float16,fp8,0,0.06603733201821645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,4,128,1,float16,float16,0,0.06756266454855601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,1,128,1,float16,float16,0,0.026682667434215546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,4,128,1,float16,fp8,0,0.0779306689898173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,48,8,128,1,float16,fp8,0,0.08998933434486389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,48,128,1,float16,float16,0,0.07670400043328603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,48,128,1,float16,fp8,0,0.07479466497898102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,1,128,1,float16,fp8,0,0.031109333038330078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,2,128,1,float16,float16,0,0.028421332438786823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,48,128,1,float16,fp8,0,0.03668266783157984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,2,128,1,float16,fp8,0,0.03254399945338567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,1,128,1,float16,float16,0,0.02032533288002014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,4,128,1,float16,float16,0,0.031013332307338715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,4,128,1,float16,fp8,0,0.03605333218971888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,8,128,1,float16,float16,0,0.03815466662247976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,48,8,128,1,float16,fp8,0,0.043354665239652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,48,128,1,float16,float16,0,0.04058666775623957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,1,128,1,float16,fp8,0,0.02372266600529353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,2,128,1,float16,float16,0,0.020181333025296528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,2,128,1,float16,fp8,0,0.023925334215164185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,4,128,1,float16,float16,0,0.021738665799299877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,4,128,1,float16,fp8,0,0.02516266703605652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,8,128,1,float16,float16,0,0.02197333425283432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,2,128,1,float16,float16,0,0.015205333630243937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,48,8,128,1,float16,fp8,0,0.025274666647116344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,48,128,1,float16,float16,0,0.02165866643190384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,48,128,1,float16,fp8,0,0.022389332453409832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,1,128,1,float16,float16,0,0.015050667027632395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,1,128,1,float16,fp8,0,0.01782400036851565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,2,128,1,float16,fp8,0,0.017477333545684814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,4,128,1,float16,float16,0,0.015333333363135656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,4,128,1,float16,fp8,0,0.017562666287024815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,8,128,1,float16,float16,0,0.01504533365368843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,1,128,1,float16,fp8,0,0.017605333278576534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,48,8,128,1,float16,fp8,0,0.01756799966096878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,48,128,1,float16,float16,0,0.016762666404247284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,48,128,1,float16,fp8,0,0.01773333301146825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,1,128,1,float16,float16,0,0.014762666076421738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,2,128,1,float16,float16,0,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,2,128,1,float16,fp8,0,0.017173333714405697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,4,128,1,float16,float16,0,0.014959999670584997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,4,128,1,float16,fp8,0,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,8,128,1,float16,float16,0,0.015125333021084467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,48,8,128,1,float16,fp8,0,0.0170666662355264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,48,128,1,float16,float16,0,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,48,128,1,float16,fp8,0,0.014122666170199713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,1,128,1,float16,float16,0,0.014314666390419006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,1,128,1,float16,fp8,0,0.016885332763195038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,8,128,1,float16,float16,0,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,2,128,1,float16,float16,0,0.014655999839305878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,2,128,1,float16,fp8,0,0.016906666258970898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,4,128,1,float16,float16,0,0.014682666709025701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,4,128,1,float16,fp8,0,0.01699200024207433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,48,8,128,1,float16,fp8,0,0.016832000265518825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,48,128,1,float16,float16,0,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,48,128,1,float16,fp8,0,0.014101333916187286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,1,128,1,float16,float16,0,0.013744000345468521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,1,128,1,float16,fp8,0,0.01569066693385442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,8,128,1,float16,fp8,0,0.01586666703224182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,2,128,1,float16,float16,0,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,2,128,1,float16,fp8,0,0.015658666690190632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,4,128,1,float16,float16,0,0.014426667243242264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,4,128,1,float16,fp8,0,0.015781333049138386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,48,8,128,1,float16,float16,0,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,2,128,1,float16,float16,0,0.10173867146174113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,1,128,1,float16,fp8,0,0.1088853379090627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,1,128,1,float16,float16,0,0.0923520028591156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,2,128,1,float16,fp8,0,0.11593066652615865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,4,128,1,float16,float16,0,0.13371200362841287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,4,128,1,float16,fp8,0,0.15799466768900552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,8,128,1,float16,float16,0,0.13945066928863525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,48,128,1,float16,float16,0,0.21874133745829263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,48,8,128,1,float16,fp8,0,0.1625440021355947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,48,128,1,float16,fp8,0,0.20877333482106528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,1,128,1,float16,float16,0,0.05478399991989136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,1,128,1,float16,fp8,0,0.06247999767462412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,2,128,1,float16,float16,0,0.059845333298047386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,2,128,1,float16,fp8,0,0.06598933537801106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,4,128,1,float16,float16,0,0.07579199969768524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,8,128,1,float16,float16,0,0.08051200211048126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,48,128,1,float16,float16,0,0.11340266466140747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,1,128,1,float16,fp8,0,0.03071466585000356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,4,128,1,float16,fp8,0,0.0883146623770396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,48,128,1,float16,fp8,0,0.11249599854151408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,48,8,128,1,float16,fp8,0,0.08966400225957234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,1,128,1,float16,float16,0,0.026837334036827087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,2,128,1,float16,float16,0,0.02809600035349528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,2,128,1,float16,fp8,0,0.032560000816980995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,4,128,1,float16,float16,0,0.035375999907652535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,4,128,1,float16,fp8,0,0.04084266722202301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,8,128,1,float16,float16,0,0.03752533346414566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,1,128,1,float16,fp8,0,0.023589332898457844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,48,8,128,1,float16,fp8,0,0.04165333261092504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,48,128,1,float16,float16,0,0.05891199906667074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,48,128,1,float16,fp8,0,0.05320000151793162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,1,128,1,float16,float16,0,0.019999999552965164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,2,128,1,float16,float16,0,0.020319999506076176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,8,128,1,float16,fp8,0,0.02550400048494339
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,2,128,1,float16,fp8,0,0.02362666775782903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,4,128,1,float16,fp8,0,0.02532266577084859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,4,128,1,float16,float16,0,0.021664001047611237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,2,128,1,float16,float16,0,0.014997333288192749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,48,8,128,1,float16,float16,0,0.021882665654023487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,48,128,1,float16,float16,0,0.030479999879995983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,4,128,1,float16,fp8,0,0.017535999417304993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,48,128,1,float16,fp8,0,0.03107733279466629
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,1,128,1,float16,float16,0,0.01492799942692121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,1,128,1,float16,fp8,0,0.01777600000301997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,2,128,1,float16,fp8,0,0.017674667139848072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,4,128,1,float16,float16,0,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,8,128,1,float16,float16,0,0.015216000378131866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,48,128,1,float16,float16,0,0.020197333147128422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,48,8,128,1,float16,fp8,0,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,48,128,1,float16,fp8,0,0.021066665649414062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,1,128,1,float16,float16,0,0.014538666854302088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,1,128,1,float16,fp8,0,0.017845333864291508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,2,128,1,float16,float16,0,0.015130666395028433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,48,128,1,float16,float16,0,0.01640533283352852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,2,128,1,float16,fp8,0,0.017605333278576534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,4,128,1,float16,float16,0,0.014933332800865173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,4,128,1,float16,fp8,0,0.016837333639462788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,2,128,1,float16,float16,0,0.0144213338692983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,8,128,1,float16,float16,0,0.014618666221698126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,48,8,128,1,float16,fp8,0,0.017162666966517765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,4,128,1,float16,float16,0,0.0143306665122509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,48,128,1,float16,fp8,0,0.017685333887736004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,8,128,1,float16,fp8,0,0.01655999943614006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,1,128,1,float16,float16,0,0.014655999839305878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,1,128,1,float16,fp8,0,0.0170666662355264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,2,128,1,float16,fp8,0,0.01691199963291486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,4,128,1,float16,fp8,0,0.01716800034046173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,1,128,1,float16,fp8,0,0.015685333559910457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,48,8,128,1,float16,float16,0,0.014373333503802618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,48,128,1,float16,float16,0,0.013280000537633896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,4,128,1,float16,float16,0,0.0138026662170887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,48,128,1,float16,fp8,0,0.01452800010641416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,1,128,1,float16,float16,0,0.013482666263977686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,2,128,1,float16,float16,0,0.013797332843144735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,2,128,1,float16,fp8,0,0.015925332903862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,4,128,1,float16,fp8,0,0.015861333658297855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,1,128,1,float16,float16,0,0.013546666751305262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,8,128,1,float16,float16,0,0.01357866699496905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,48,8,128,1,float16,fp8,0,0.01571200042963028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,48,128,1,float16,float16,0,0.013295999417702356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,4,128,1,float16,float16,0,0.013765333841244379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,4,128,1,float16,fp8,0,0.015568000574906668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,48,128,1,float16,fp8,0,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,1,128,1,float16,fp8,0,0.01573333392540614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,2,128,1,float16,float16,0,0.013471999516089758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,2,128,1,float16,fp8,0,0.015647999942302704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,8,128,1,float16,float16,0,0.013663999736309052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,48,8,128,1,float16,fp8,0,0.015423999478419622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,0,0.022309333086013794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,0,0.02436800052722295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,0,0.029071999092896778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,0,0.030928000807762146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,0,0.04219200213750204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,0,0.044362664222717285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,0,0.06673599779605865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,0,0.07100800176461537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,48,128,1,float16,float16,0,0.1646453340848287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,48,128,1,float16,fp8,0,0.17578667402267456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,0,0.014970666418472925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,0,0.019695999721686046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,0,0.016352000335852306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,0,0.01876266673207283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,0,0.027093333502610523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,0,0.025199999411900837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,0,0.03766933331886927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,0,0.041120000183582306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,0,0.01321600005030632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,48,128,1,float16,float16,0,0.08730133374532063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,48,128,1,float16,fp8,0,0.09311466415723164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,0,0.012954667210578918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,0,0.02378133436044057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,0,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,0,0.01413333291808764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,48,128,1,float16,fp8,0,0.05484800040721893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,0,0.01661866654952367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,0,0.018239999810854595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,0,0.025360000630219776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,48,128,1,float16,float16,0,0.04942933221658071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,0,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,0,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,0,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,0,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,0,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,0,0.013653332988421122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,0,0.016373333831628162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,0,0.017322666943073273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,48,128,1,float16,float16,0,0.03018666555484136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,48,128,1,float16,fp8,0,0.03302400062481562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,0,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,0,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,0,0.011802667131026586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,0,0.012730666746695837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,0,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,0,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,0,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,0,0.013397333522637686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,48,128,1,float16,float16,0,0.020245333512624104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,48,128,1,float16,fp8,0,0.022405333817005157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,0,0.011018666128317514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,0,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,0,0.013082666943470636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,0,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,48,128,1,float16,fp8,0,0.017914666483799618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,0,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,0,0.011770666887362799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,0,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,0,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,0,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,48,128,1,float16,float16,0,0.015941333025693893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,0,0.010949333508809408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,0,0.011482667177915573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,0,0.011343999455372492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,0,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,0,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,0,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,0,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,48,128,1,float16,float16,0,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,48,128,1,float16,fp8,0,0.014597332725922266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,0,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,0,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,0,0.012543999900420507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,0,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,0,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,0,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,0,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,0,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,48,128,1,float16,float16,0,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,48,128,1,float16,fp8,0,0.01423466702302297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,0,0.011173332730929056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,0,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,0,0.01129066695769628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,0,0.01110400011142095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,0,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,0,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,0,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,0,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,1,128,1,float16,fp8,0,34.21020762125651
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,2,128,1,float16,fp8,0,34.209765116373696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,1,128,1,float16,float16,0,38.35226186116537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,4,128,1,float16,fp8,0,34.940826416015625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,2,128,1,float16,float16,0,38.507771809895836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,4,128,1,float16,float16,0,39.060864766438804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,8,128,1,float16,float16,0,38.675008138020836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,40,128,1,float16,fp8,0,18.3462397257487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,40,128,1,float16,float16,0,20.484933217366535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,1,128,1,float16,fp8,0,17.077471415201824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,1,128,1,float16,float16,0,18.86070378621419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,2,128,1,float16,fp8,0,17.19306182861328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,2,128,1,float16,float16,0,19.53124237060547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,4,128,1,float16,float16,0,19.2970453898112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,40,128,1,float16,float16,0,10.607840220133463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,40,128,1,float16,fp8,0,9.241573333740234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,1,128,1,float16,float16,0,9.490293502807617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,1,128,1,float16,fp8,0,8.617989222208658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,4,128,1,float16,fp8,0,17.73464584350586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,40,8,128,1,float16,fp8,0,65.12654622395833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,8,128,1,float16,fp8,0,17.68872578938802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,40,8,128,1,float16,float16,0,20.440821329752605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,2,128,1,float16,fp8,0,9.157679875691732
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,2,128,1,float16,float16,0,11.088277180989584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,40,128,1,float16,float16,0,5.139674822489421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,4,128,1,float16,float16,0,11.39090092976888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,4,128,1,float16,fp8,0,10.120901107788086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,40,128,1,float16,fp8,0,5.537845611572266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,1,128,1,float16,float16,0,4.6113332112630205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,8,128,1,float16,float16,0,11.484539031982422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,1,128,1,float16,fp8,0,4.513221422831218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,2,128,1,float16,float16,0,5.251706759134929
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,4,128,1,float16,float16,0,4.804650624593099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,2,128,1,float16,fp8,0,4.945045471191406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,4,128,1,float16,fp8,0,4.494367917378743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,40,8,128,1,float16,fp8,0,14.856975555419922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,8,128,1,float16,float16,0,5.966367721557617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,40,8,128,1,float16,fp8,0,4.714463869730632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,1,128,1,float16,fp8,0,19.52684783935547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,2,128,1,float16,fp8,0,19.699498494466145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,1,128,1,float16,float16,0,22.12962595621745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,2,128,1,float16,float16,0,22.98102315266927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,4,128,1,float16,float16,0,22.50709279378255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,8,128,1,float16,fp8,0,20.27259699503581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,8,128,1,float16,float16,0,22.590240478515625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,40,128,1,float16,fp8,0,10.75829823811849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,40,128,1,float16,float16,0,12.267743428548178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,1,128,1,float16,float16,0,11.103205362955729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,1,128,1,float16,fp8,0,9.8079465230306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,2,128,1,float16,float16,0,11.216341654459635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,2,128,1,float16,fp8,0,9.852837244669596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,40,4,128,1,float16,fp8,0,34.936180114746094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,4,128,1,float16,float16,0,11.839738210042318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,40,128,1,float16,float16,0,5.691466649373372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,1,128,1,float16,float16,0,5.238602638244629
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,40,128,1,float16,fp8,0,5.652352015177409
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,4,128,1,float16,fp8,0,10.170901616414389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,8,128,1,float16,float16,0,10.97311528523763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,40,8,128,1,float16,fp8,0,10.200991948445639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,2,128,1,float16,float16,0,5.338464101155599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,1,128,1,float16,fp8,0,8.493551890055338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,2,128,1,float16,fp8,0,5.169210751851399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,4,128,1,float16,fp8,0,5.135210673014323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,4,128,1,float16,float16,0,5.591328303019206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,40,128,1,float16,float16,0,2.762960116068522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,8,128,1,float16,float16,0,5.366218566894531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,40,128,1,float16,fp8,0,3.0138559341430664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,1,128,1,float16,float16,0,5.978143692016602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,1,128,1,float16,fp8,0,2.596656004587809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,40,8,128,1,float16,fp8,0,5.287397384643555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,2,128,1,float16,float16,0,2.5892106691996255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,8,128,1,float16,float16,0,2.6538027127583823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,2,128,1,float16,fp8,0,2.707493464152018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,4,128,1,float16,float16,0,2.6124374071756997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,4,128,1,float16,fp8,0,2.8922878901163735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,40,8,128,1,float16,fp8,0,4.486261367797852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,1,128,1,float16,fp8,0,13.76046371459961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,1,128,1,float16,float16,0,15.696975708007812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,2,128,1,float16,fp8,0,13.88430913289388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,2,128,1,float16,float16,0,15.832224527994791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,4,128,1,float16,fp8,0,14.3591677347819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,4,128,1,float16,float16,0,16.21324284871419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,8,128,1,float16,float16,0,16.26096471150716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,40,128,1,float16,float16,0,8.542191823323568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,40,128,1,float16,fp8,0,7.761525472005208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,1,128,1,float16,float16,0,7.746981302897136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,1,128,1,float16,fp8,0,6.933551788330078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,2,128,1,float16,float16,0,7.087253570556641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,2,128,1,float16,fp8,0,7.000661214192708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,4,128,1,float16,float16,0,8.127893447875977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,40,128,1,float16,float16,0,3.9182186126708984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,40,128,1,float16,fp8,0,3.891434669494629
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,1,128,1,float16,float16,0,3.9084161122639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,4,128,1,float16,fp8,0,7.252400080362956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,40,8,128,1,float16,fp8,0,28.189854939778645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,8,128,1,float16,float16,0,8.08299191792806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,40,8,128,1,float16,fp8,0,7.5778452555338545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,2,128,1,float16,float16,0,3.661439895629883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,1,128,1,float16,fp8,0,3.971424102783203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,2,128,1,float16,fp8,0,3.9919681549072266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,4,128,1,float16,float16,0,3.6778186162312827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,8,128,1,float16,float16,0,3.799994786580404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,40,128,1,float16,float16,0,2.0830772717793784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,8,128,1,float16,fp8,0,3.6698878606160483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,40,128,1,float16,fp8,0,2.0106186866760254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,1,128,1,float16,float16,0,1.7929813067118328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,40,4,128,1,float16,fp8,0,7.027381261189778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,1,128,1,float16,fp8,0,1.900485356648763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,2,128,1,float16,float16,0,1.9352693557739258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,2,128,1,float16,fp8,0,1.9468960762023926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,4,128,1,float16,float16,0,1.8584532737731934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,4,128,1,float16,fp8,0,1.8729920387268066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,8,128,1,float16,fp8,0,1.8821652730305989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,40,8,128,1,float16,float16,0,1.8690667152404785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,1,128,1,float16,fp8,0,17.971200307210285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,2,128,1,float16,fp8,0,18.10985565185547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,2,128,1,float16,float16,0,19.83346176147461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,4,128,1,float16,fp8,0,18.853599548339844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,4,128,1,float16,float16,0,20.60260772705078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,8,128,1,float16,float16,0,20.836048126220703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,8,128,1,float16,fp8,0,18.96673075358073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,40,128,1,float16,float16,0,11.358634948730469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,1,128,1,float16,float16,0,10.212618509928385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,40,128,1,float16,fp8,0,10.266698837280273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,1,128,1,float16,fp8,0,9.054709116617838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,2,128,1,float16,float16,0,10.484986623128256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,2,128,1,float16,fp8,0,9.098917643229166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,4,128,1,float16,float16,0,10.47539202372233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,40,128,1,float16,float16,0,5.212725321451823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,1,128,1,float16,float16,0,4.888789176940918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,40,128,1,float16,fp8,0,5.6501922607421875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,1,128,1,float16,fp8,0,4.580410639444987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,4,128,1,float16,fp8,0,9.515951792399088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,8,128,1,float16,fp8,0,9.451290766398111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,40,8,128,1,float16,float16,0,10.467173258463541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,2,128,1,float16,float16,0,5.002026557922363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,2,128,1,float16,fp8,0,4.783103942871094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,4,128,1,float16,float16,0,4.8659467697143555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,40,128,1,float16,float16,0,2.616480032602946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,4,128,1,float16,fp8,0,4.775317192077637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,8,128,1,float16,float16,0,4.835573196411133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,40,8,128,1,float16,fp8,0,4.798122723897298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,1,128,1,float16,float16,0,2.3132479985555015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,40,128,1,float16,fp8,0,2.7923253377278647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,2,128,1,float16,float16,0,2.3561174074808755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,1,128,1,float16,fp8,0,2.8971786499023438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,2,128,1,float16,fp8,0,2.316805362701416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,4,128,1,float16,float16,0,2.5080533027648926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,4,128,1,float16,fp8,0,2.4233333269755044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,40,128,1,float16,float16,0,1.3550666173299153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,8,128,1,float16,float16,0,2.462437311808268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,40,128,1,float16,fp8,0,1.4160586992899578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,40,8,128,1,float16,fp8,0,2.512320041656494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,1,128,1,float16,float16,0,1.173962672551473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,1,128,1,float16,fp8,0,1.2901546955108643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,2,128,1,float16,float16,0,1.1932266553243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,2,128,1,float16,fp8,0,1.2737332979838054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,4,128,1,float16,float16,0,1.2528746922810872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,4,128,1,float16,fp8,0,1.2607466379801433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,8,128,1,float16,float16,0,1.252832015355428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,40,8,128,1,float16,fp8,0,1.2889066537221272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,40,1,128,1,float16,float16,0,67.52968343098958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,1,128,1,float16,float16,0,11.434575398763021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,1,128,1,float16,fp8,0,10.520757039388021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,2,128,1,float16,float16,0,11.729199727376303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,2,128,1,float16,fp8,0,10.735764821370443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,4,128,1,float16,float16,0,12.412704467773438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,4,128,1,float16,fp8,0,11.199626922607422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,8,128,1,float16,float16,0,12.510735829671225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,40,128,1,float16,float16,0,6.351173400878906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,40,128,1,float16,fp8,0,6.183088302612305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,1,128,1,float16,float16,0,5.659621556599935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,1,128,1,float16,fp8,0,5.2640425364176435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,2,128,1,float16,float16,0,5.851589202880859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,2,128,1,float16,fp8,0,5.306138674418132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,4,128,1,float16,float16,0,5.822160085042317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,40,128,1,float16,float16,0,3.091813405354818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,1,128,1,float16,float16,0,2.7150611877441406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,40,128,1,float16,fp8,0,3.4595521291097007
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,8,128,1,float16,float16,0,6.150714874267578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,4,128,1,float16,fp8,0,5.865125020345052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,40,8,128,1,float16,fp8,0,5.85528564453125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,1,128,1,float16,fp8,0,2.933621406555176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,2,128,1,float16,float16,0,2.8970505396525064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,2,128,1,float16,fp8,0,2.836218516031901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,4,128,1,float16,float16,0,2.8033227920532227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,4,128,1,float16,fp8,0,3.119482676188151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,40,128,1,float16,float16,0,1.5698986053466797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,8,128,1,float16,float16,0,2.7986186345418296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,40,128,1,float16,fp8,0,1.7530666987101238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,40,8,128,1,float16,fp8,0,2.8623733520507812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,1,128,1,float16,float16,0,1.3532959620157878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,1,128,1,float16,fp8,0,1.3873119354248047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,2,128,1,float16,float16,0,1.3753493626912434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,40,8,128,1,float16,fp8,0,26.959462483723957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,2,128,1,float16,fp8,0,1.439509391784668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,4,128,1,float16,fp8,0,1.4547306696573894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,4,128,1,float16,float16,0,1.4285386403401692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,8,128,1,float16,float16,0,1.4431947072347004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,40,8,128,1,float16,fp8,0,1.4938294092814128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,40,128,1,float16,float16,0,0.8154239654541016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,40,128,1,float16,fp8,0,0.9921279748280843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,1,128,1,float16,float16,0,0.6875360012054443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,1,128,1,float16,fp8,0,0.7098080317179362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,2,128,1,float16,float16,0,0.7074080308278402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,2,128,1,float16,fp8,0,0.7350026766459147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,4,128,1,float16,float16,0,0.7331786950429281
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,4,128,1,float16,fp8,0,0.7755786577860514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,8,128,1,float16,float16,0,0.944373369216919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,40,8,128,1,float16,fp8,0,0.7802773316701254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,1,128,1,float16,float16,0,10.840260823567709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,1,128,1,float16,fp8,0,9.860095977783203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,2,128,1,float16,float16,0,10.923802693684896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,2,128,1,float16,fp8,0,10.107248306274414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,4,128,1,float16,float16,0,11.757867177327475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,4,128,1,float16,fp8,0,10.900421142578125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,8,128,1,float16,fp8,0,10.953914642333984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,1,128,1,float16,fp8,0,4.958741188049316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,1,128,1,float16,float16,0,5.309520085652669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,40,128,1,float16,float16,0,6.117247899373372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,40,128,1,float16,fp8,0,6.1644852956136065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,2,128,1,float16,float16,0,5.07807985941569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,2,128,1,float16,fp8,0,5.061829249064128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,4,128,1,float16,float16,0,5.534528096516927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,40,128,1,float16,float16,0,3.044901212056478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,1,128,1,float16,float16,0,2.490351994832357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,4,128,1,float16,fp8,0,5.467397054036458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,40,128,1,float16,fp8,0,3.3154452641805015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,8,128,1,float16,float16,0,5.608421325683594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,40,8,128,1,float16,fp8,0,5.7327626546223955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,2,128,1,float16,float16,0,2.5725173950195312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,1,128,1,float16,fp8,0,2.837717374165853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,2,128,1,float16,fp8,0,2.659226735432943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,4,128,1,float16,float16,0,2.6925598780314126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,4,128,1,float16,fp8,0,2.803167978922526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,40,8,128,1,float16,float16,0,29.57379150390625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,8,128,1,float16,float16,0,2.7315572102864585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,1,128,1,float16,float16,0,1.246677319208781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,40,128,1,float16,float16,0,1.5531306266784668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,40,8,128,1,float16,fp8,0,2.8345279693603516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,40,128,1,float16,fp8,0,1.62990935643514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,1,128,1,float16,fp8,0,1.3731412887573242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,4,128,1,float16,float16,0,1.3620212872823079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,2,128,1,float16,float16,0,1.2861173152923584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,4,128,1,float16,fp8,0,1.4296372731526692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,2,128,1,float16,fp8,0,2.1397172609965005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,8,128,1,float16,float16,0,1.3743947347005208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,40,8,128,1,float16,fp8,0,1.4241120020548503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,40,128,1,float16,float16,0,0.7940053145090739
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,40,128,1,float16,fp8,0,0.8479680220286051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,1,128,1,float16,float16,0,0.6421066522598267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,1,128,1,float16,fp8,0,0.6769279638926188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,2,128,1,float16,float16,0,0.6597599983215332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,4,128,1,float16,float16,0,0.6952959696451823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,2,128,1,float16,fp8,0,0.9264427026112875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,4,128,1,float16,fp8,0,0.7275839646657308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,8,128,1,float16,float16,0,0.7108266353607178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,40,8,128,1,float16,fp8,0,0.7520053386688232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,40,128,1,float16,float16,0,0.420357346534729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,40,128,1,float16,fp8,0,0.44973333676656085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,1,128,1,float16,float16,0,0.34140801429748535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,1,128,1,float16,fp8,0,0.3612639904022217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,2,128,1,float16,float16,0,0.3726346492767334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,2,128,1,float16,fp8,0,0.36265599727630615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,4,128,1,float16,float16,0,0.3659520149230957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,4,128,1,float16,fp8,0,0.3935146729151408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,8,128,1,float16,float16,0,0.37434665362040204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,40,8,128,1,float16,fp8,0,0.40163731575012207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,1,128,1,float16,float16,0,6.357983907063802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,2,128,1,float16,fp8,0,6.057210922241211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,4,128,1,float16,float16,0,6.640111923217773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,4,128,1,float16,fp8,0,6.711530685424805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,8,128,1,float16,float16,0,6.688394546508789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,1,128,1,float16,fp8,0,6.063141504923503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,8,128,1,float16,fp8,0,6.756650924682617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,40,128,1,float16,float16,0,3.8013172149658203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,40,2,128,1,float16,float16,0,10.557466506958008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,1,128,1,float16,float16,0,3.167125384012858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,40,128,1,float16,fp8,0,3.917125384012858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,1,128,1,float16,fp8,0,2.993781407674154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,2,128,1,float16,float16,0,3.0690666834513345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,2,128,1,float16,fp8,0,3.031578699747721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,4,128,1,float16,float16,0,3.2955039342244468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,40,128,1,float16,float16,0,1.8784960110982258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,4,128,1,float16,fp8,0,3.4545119603474936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,40,128,1,float16,fp8,0,1.995456059773763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,1,128,1,float16,float16,0,1.4722026189168294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,8,128,1,float16,fp8,0,3.503216107686361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,1,128,1,float16,fp8,0,1.507263978322347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,40,8,128,1,float16,float16,0,5.6003468831380205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,2,128,1,float16,float16,0,1.5627039273579915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,2,128,1,float16,fp8,0,1.587839921315511
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,4,128,1,float16,float16,0,1.6251200040181477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,4,128,1,float16,fp8,0,1.693002700805664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,8,128,1,float16,float16,0,1.6433226267496746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,40,128,1,float16,fp8,0,1.0951680342356365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,40,8,128,1,float16,fp8,0,1.7104640007019043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,40,128,1,float16,float16,0,1.0732853412628174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,1,128,1,float16,float16,0,0.7427199681599935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,2,128,1,float16,float16,0,0.7692106564839681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,1,128,1,float16,fp8,0,0.7952906290690104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,4,128,1,float16,fp8,0,0.8672693570454916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,2,128,1,float16,fp8,0,0.8113706906636556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,4,128,1,float16,float16,0,0.8298559983571371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,8,128,1,float16,float16,0,0.9154667059580485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,40,128,1,float16,float16,0,0.49697065353393555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,40,8,128,1,float16,fp8,0,0.9072373708089193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,40,128,1,float16,fp8,0,0.547050674756368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,1,128,1,float16,float16,0,0.3839840094248454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,1,128,1,float16,fp8,0,0.4170080025990804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,8,128,1,float16,float16,0,0.43721067905426025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,2,128,1,float16,float16,0,0.40514131387074787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,2,128,1,float16,fp8,0,0.487557331720988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,4,128,1,float16,float16,0,0.43057068188985187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,8,128,1,float16,fp8,0,0.47518400351206463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,40,128,1,float16,float16,0,0.2662453254063924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,40,4,128,1,float16,fp8,0,0.45929598808288574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,40,128,1,float16,fp8,0,0.2956053415934245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,1,128,1,float16,float16,0,0.21426665782928467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,1,128,1,float16,fp8,0,0.23305066426595053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,2,128,1,float16,float16,0,0.22090667486190796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,2,128,1,float16,fp8,0,0.2376586596171061
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,4,128,1,float16,float16,0,0.23135999838511148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,4,128,1,float16,fp8,0,0.2478933334350586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,8,128,1,float16,float16,0,0.2355466683705648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,40,8,128,1,float16,fp8,0,0.25939732789993286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,1,128,1,float16,float16,0,5.568933486938477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,1,128,1,float16,fp8,0,5.964517593383789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,4,128,1,float16,float16,0,6.75212287902832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,2,128,1,float16,fp8,0,6.089621225992839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,4,128,1,float16,fp8,0,6.921242396036784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,8,128,1,float16,float16,0,6.712490717569987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,8,128,1,float16,fp8,0,6.850837071736653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,40,128,1,float16,float16,0,3.949061393737793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,40,128,1,float16,fp8,0,4.226442654927571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,1,128,1,float16,float16,0,2.8927841186523438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,1,128,1,float16,fp8,0,2.9745492935180664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,2,128,1,float16,float16,0,2.9514506657918296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,2,128,1,float16,fp8,0,3.19046942392985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,4,128,1,float16,float16,0,3.2854134241739907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,4,128,1,float16,fp8,0,3.6059252421061196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,40,128,1,float16,float16,0,1.9951626459757488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,8,128,1,float16,float16,0,3.3564799626668296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,40,128,1,float16,fp8,0,2.2104320526123047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,1,128,1,float16,float16,0,1.414522647857666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,40,2,128,1,float16,float16,0,20.316735585530598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,1,128,1,float16,fp8,0,1.545141379038493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,40,8,128,1,float16,fp8,0,3.5672105153401694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,4,128,1,float16,float16,0,1.6581066449483235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,2,128,1,float16,float16,0,1.481818675994873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,2,128,1,float16,fp8,0,1.567482630411784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,4,128,1,float16,fp8,0,1.789514700571696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,8,128,1,float16,float16,0,1.6499253908793132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,40,128,1,float16,float16,0,1.0155413150787354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,1,128,1,float16,float16,0,0.730288028717041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,40,128,1,float16,fp8,0,1.1078879833221436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,1,128,1,float16,fp8,0,0.784821351369222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,2,128,1,float16,float16,0,0.76528533299764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,40,8,128,1,float16,fp8,0,3.6782185236612954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,2,128,1,float16,fp8,0,0.7931626637776693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,4,128,1,float16,float16,0,0.8300053278605143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,4,128,1,float16,fp8,0,0.9118773142496744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,8,128,1,float16,float16,0,0.8423733711242676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,40,8,128,1,float16,fp8,0,0.9281973044077555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,40,128,1,float16,float16,0,0.5195306539535522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,40,128,1,float16,fp8,0,0.5716906785964966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,4,128,1,float16,float16,0,0.4323413372039795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,1,128,1,float16,float16,0,0.39637335141499835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,1,128,1,float16,fp8,0,0.40436800320943195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,2,128,1,float16,float16,0,0.3958666721979777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,2,128,1,float16,fp8,0,0.4201333522796631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,4,128,1,float16,fp8,0,0.46827733516693115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,8,128,1,float16,float16,0,0.4369279940923055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,40,128,1,float16,float16,0,0.2787359952926636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,40,8,128,1,float16,fp8,0,0.47915732860565186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,40,128,1,float16,fp8,0,0.29981333017349243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,1,128,1,float16,float16,0,0.20171199242273966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,1,128,1,float16,fp8,0,0.22246400515238443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,2,128,1,float16,float16,0,0.2153493364651998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,40,128,1,float16,float16,0,0.15396266182263693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,2,128,1,float16,fp8,0,0.22523200511932373
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,4,128,1,float16,float16,0,0.23147734006245932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,4,128,1,float16,fp8,0,0.25377599398295086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,1,128,1,float16,fp8,0,0.12303466598192851
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,8,128,1,float16,float16,0,0.2373653252919515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,40,8,128,1,float16,fp8,0,0.29897600412368774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,40,128,1,float16,fp8,0,0.1702293356259664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,1,128,1,float16,float16,0,0.11423466602961223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,8,128,1,float16,fp8,0,0.14847999811172485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,2,128,1,float16,float16,0,0.1190826694170634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,2,128,1,float16,fp8,0,0.12799466649691263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,4,128,1,float16,float16,0,0.12914666533470154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,4,128,1,float16,fp8,0,0.14074666301409403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,40,8,128,1,float16,float16,0,0.13326399525006613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,1,128,1,float16,float16,0,3.4324000676472983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,2,128,1,float16,float16,0,3.5724426905314126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,2,128,1,float16,fp8,0,3.861130714416504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,4,128,1,float16,float16,0,4.083418528238933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,8,128,1,float16,float16,0,4.115658760070801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,1,128,1,float16,fp8,0,11.6603151957194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,1,128,1,float16,float16,0,1.778773307800293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,4,128,1,float16,fp8,0,4.434288024902344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,40,128,1,float16,float16,0,2.5766666730244956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,40,8,128,1,float16,fp8,0,4.605456034342448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,40,128,1,float16,fp8,0,2.850229263305664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,1,128,1,float16,fp8,0,1.941941261291504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,2,128,1,float16,fp8,0,1.960741360982259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,4,128,1,float16,float16,0,2.027397314707438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,2,128,1,float16,float16,0,4.529226620992024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,4,128,1,float16,fp8,0,2.2683092753092446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,8,128,1,float16,float16,0,2.0613706906636557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,1,128,1,float16,float16,0,0.880570650100708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,40,8,128,1,float16,fp8,0,2.277034600575765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,40,128,1,float16,float16,0,1.3061493237813313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,1,128,1,float16,fp8,0,0.9479626814524332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,4,128,1,float16,float16,0,1.0248586336771648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,4,128,1,float16,fp8,0,1.156447966893514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,2,128,1,float16,fp8,0,0.9955039819081625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,8,128,1,float16,float16,0,1.0357173283894856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,2,128,1,float16,float16,0,2.961141268412272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,40,128,1,float16,fp8,0,0.7333119710286459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,40,128,1,float16,float16,0,0.662224014600118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,1,128,1,float16,float16,0,0.4570986827214559
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,8,128,1,float16,fp8,0,1.1718826293945312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,40,40,128,1,float16,fp8,0,1.454426606496175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,1,128,1,float16,fp8,0,0.49612800280253094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,2,128,1,float16,float16,0,0.47175467014312744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,2,128,1,float16,fp8,0,0.8683679898579916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,4,128,1,float16,float16,0,0.5362720092137655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,8,128,1,float16,float16,0,0.5453173319498698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,4,128,1,float16,fp8,0,0.5943093299865723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,40,8,128,1,float16,fp8,0,0.6078986724217733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,1,128,1,float16,float16,0,0.2364693284034729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,40,128,1,float16,fp8,0,0.386624018351237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,40,128,1,float16,float16,0,0.3506133159001668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,1,128,1,float16,fp8,0,0.3034133315086365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,2,128,1,float16,fp8,0,0.27552000681559247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,2,128,1,float16,float16,0,0.2544426719347636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,4,128,1,float16,float16,0,0.28244266907374066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,4,128,1,float16,fp8,0,0.3133653402328491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,8,128,1,float16,float16,0,0.2821813424428304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,40,8,128,1,float16,fp8,0,0.315120001633962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,40,128,1,float16,float16,0,0.18757333358128866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,40,128,1,float16,fp8,0,0.2098346749941508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,1,128,1,float16,float16,0,0.13363200426101685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,1,128,1,float16,fp8,0,0.14780799547831217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,2,128,1,float16,float16,0,0.13934399684270224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,8,128,1,float16,fp8,0,0.1741973360379537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,2,128,1,float16,fp8,0,0.15271466970443726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,4,128,1,float16,float16,0,0.14990400274594626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,4,128,1,float16,fp8,0,0.17005334297815958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,2,128,1,float16,float16,0,0.07720533510049184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,40,8,128,1,float16,float16,0,0.15685866276423135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,40,128,1,float16,float16,0,0.10645332932472229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,40,128,1,float16,fp8,0,0.12121599912643433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,1,128,1,float16,float16,0,0.07442666590213776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,8,128,1,float16,fp8,0,0.09773866335550944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,1,128,1,float16,fp8,0,0.08025600016117096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,2,128,1,float16,fp8,0,0.0839573343594869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,4,128,1,float16,float16,0,0.08543466528256734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,4,128,1,float16,fp8,0,0.09334933757781982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,40,8,128,1,float16,float16,0,0.08855467041333516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,1,128,1,float16,float16,0,3.5808693567911782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,2,128,1,float16,float16,0,3.8070240020751953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,1,128,1,float16,fp8,0,3.988832155863444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,8,128,1,float16,float16,0,4.407951990763347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,8,128,1,float16,fp8,0,4.969312032063802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,40,128,1,float16,float16,0,2.9106454849243164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,2,128,1,float16,fp8,0,8.473040262858072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,4,128,1,float16,float16,0,4.38153076171875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,1,128,1,float16,float16,0,1.7841599782307942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,40,4,128,1,float16,fp8,0,4.9432268142700195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,40,128,1,float16,fp8,0,3.3501707712809243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,1,128,1,float16,fp8,0,2.0316640535990396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,2,128,1,float16,float16,0,1.8926666577657063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,2,128,1,float16,fp8,0,2.103509267171224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,8,128,1,float16,float16,0,2.206554730733236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,4,128,1,float16,fp8,0,2.5312746365865073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,40,128,1,float16,float16,0,1.4692427317301433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,1,128,1,float16,float16,0,0.9074239730834961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,4,128,1,float16,float16,0,4.2092587153116865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,40,128,1,float16,fp8,0,1.6757706006368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,40,8,128,1,float16,fp8,0,2.502336025238037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,1,128,1,float16,fp8,0,1.0295946598052979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,2,128,1,float16,float16,0,0.9886559645334879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,2,128,1,float16,fp8,0,1.0490559736887615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,4,128,1,float16,float16,0,1.129701296488444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,4,128,1,float16,fp8,0,1.2699360052744548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,40,128,1,float16,float16,0,0.7443199952443441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,8,128,1,float16,fp8,0,1.3155360221862793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,40,8,128,1,float16,float16,0,1.3092906475067139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,40,128,1,float16,fp8,0,0.8590559959411621
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,1,128,1,float16,float16,0,0.46930134296417236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,1,128,1,float16,fp8,0,0.518453319867452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,2,128,1,float16,float16,0,0.515941341718038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,2,128,1,float16,fp8,0,0.5465279817581177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,4,128,1,float16,float16,0,0.580736001332601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,4,128,1,float16,fp8,0,0.6891252994537354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,8,128,1,float16,float16,0,0.5830719868342081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,1,128,1,float16,float16,0,0.24895467360814413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,40,8,128,1,float16,fp8,0,0.664250651995341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,40,128,1,float16,float16,0,0.38492266337076825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,40,128,1,float16,fp8,0,0.44833068052927655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,4,128,1,float16,float16,0,0.3054400086402893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,1,128,1,float16,fp8,0,0.2773173252741496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,2,128,1,float16,float16,0,0.2646026611328125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,2,128,1,float16,fp8,0,0.2858399947484334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,4,128,1,float16,fp8,0,0.33839468161265057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,8,128,1,float16,float16,0,0.3043946623802185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,40,8,128,1,float16,fp8,0,0.3516266743342082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,40,128,1,float16,float16,0,0.21146667003631592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,40,128,1,float16,fp8,0,0.23830399910608926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,1,128,1,float16,float16,0,0.13397333025932312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,1,128,1,float16,fp8,0,0.14991999665896097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,2,128,1,float16,fp8,0,0.15754666924476624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,2,128,1,float16,float16,0,0.14572266737620035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,4,128,1,float16,float16,0,0.16204800208409628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,4,128,1,float16,fp8,0,0.18216000000635782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,8,128,1,float16,fp8,0,0.18868800004323324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,40,8,128,1,float16,float16,0,0.16711467504501343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,40,128,1,float16,float16,0,0.11442133784294128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,4,128,1,float16,float16,0,0.09114666779836018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,2,128,1,float16,fp8,0,0.09172266721725464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,40,128,1,float16,fp8,0,0.13210133711496988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,1,128,1,float16,float16,0,0.07700266440709432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,1,128,1,float16,fp8,0,0.08515200018882751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,2,128,1,float16,float16,0,0.08021333316961925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,4,128,1,float16,fp8,0,0.10181333621342976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,8,128,1,float16,float16,0,0.09193066755930583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,40,8,128,1,float16,fp8,0,0.107205331325531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,40,128,1,float16,float16,0,0.06764799853165944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,40,128,1,float16,fp8,0,0.07881600161393483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,1,128,1,float16,float16,0,0.0451200008392334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,1,128,1,float16,fp8,0,0.04976533353328705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,2,128,1,float16,float16,0,0.04684266448020935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,2,128,1,float16,fp8,0,0.054378668467203774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,4,128,1,float16,float16,0,0.050069332122802734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,4,128,1,float16,fp8,0,0.05552533268928528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,8,128,1,float16,float16,0,0.050437331199645996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,40,8,128,1,float16,fp8,0,0.0591786652803421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,1,128,1,float16,float16,0,2.656405289967855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,2,128,1,float16,float16,0,2.897594769795736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,1,128,1,float16,fp8,0,3.0202134450276694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,2,128,1,float16,fp8,0,3.1965014139811196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,1,128,1,float16,float16,0,1.330906629562378
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,40,128,1,float16,float16,0,2.426431973775228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,4,128,1,float16,fp8,0,4.029280026753743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,4,128,1,float16,float16,0,7.098405202229817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,1,128,1,float16,fp8,0,1.5219680468241374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,40,128,1,float16,fp8,0,2.954773267110189
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,2,128,1,float16,float16,0,1.4553546905517578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,2,128,1,float16,fp8,0,1.6002027193705242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,8,128,1,float16,float16,0,3.5284852981567383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,40,8,128,1,float16,fp8,0,4.088992118835449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,4,128,1,float16,float16,0,1.74345064163208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,40,128,1,float16,float16,0,1.224117358525594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,4,128,1,float16,fp8,0,2.39027738571167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,40,128,1,float16,fp8,0,1.506325403849284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,8,128,1,float16,float16,0,1.7511946360270183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,40,8,128,1,float16,fp8,0,2.0557173093159995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,1,128,1,float16,float16,0,0.6747732957204183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,2,128,1,float16,float16,0,0.7387306690216064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,2,128,1,float16,fp8,0,0.8146080176035563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,40,128,1,float16,float16,0,0.6226346492767334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,4,128,1,float16,float16,0,0.8881440162658691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,40,128,1,float16,fp8,0,0.7599360148111979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,8,128,1,float16,fp8,0,1.0723360379536946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,4,128,1,float16,fp8,0,1.0301226774851482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,1,128,1,float16,float16,0,0.35037867228190106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,1,128,1,float16,fp8,0,0.7727680206298828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,1,128,1,float16,fp8,0,0.398037314414978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,2,128,1,float16,float16,0,0.3855573336283366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,40,8,128,1,float16,float16,0,0.903493324915568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,4,128,1,float16,fp8,0,0.5291893482208252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,2,128,1,float16,fp8,0,0.41890132427215576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,4,128,1,float16,float16,0,0.4472479820251465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,8,128,1,float16,fp8,0,0.5482133229573568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,1,128,1,float16,float16,0,0.18494399388631186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,2,128,1,float16,float16,0,0.20149334271748862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,40,128,1,float16,float16,0,0.32680533329645794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,40,128,1,float16,fp8,0,0.39419734477996826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,1,128,1,float16,fp8,0,0.21013865868250528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,40,8,128,1,float16,float16,0,0.46902934710184735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,2,128,1,float16,fp8,0,0.22265066703160605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,4,128,1,float16,float16,0,0.23553067445755005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,4,128,1,float16,fp8,0,0.27581334114074707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,8,128,1,float16,float16,0,0.24342399835586548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,40,128,1,float16,float16,0,0.17618133624394736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,40,128,1,float16,fp8,0,0.21009600162506104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,40,8,128,1,float16,fp8,0,0.2885119915008545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,1,128,1,float16,fp8,0,0.1163200040658315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,1,128,1,float16,float16,0,0.1023573378721873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,2,128,1,float16,float16,0,0.11058132847150166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,2,128,1,float16,fp8,0,0.12361600001653035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,4,128,1,float16,float16,0,0.1270080010096232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,4,128,1,float16,fp8,0,0.14855999747912088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,8,128,1,float16,float16,0,0.12991467118263245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,40,8,128,1,float16,fp8,0,0.15611732999483743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,40,128,1,float16,float16,0,0.09711466232935588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,40,128,1,float16,fp8,0,0.11725866794586182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,2,128,1,float16,fp8,0,0.07214400172233582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,1,128,1,float16,float16,0,0.06010666489601135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,1,128,1,float16,fp8,0,0.06665066878000896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,8,128,1,float16,float16,0,0.07317866881688435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,2,128,1,float16,float16,0,0.06423999865849812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,4,128,1,float16,float16,0,0.07148266832033794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,4,128,1,float16,fp8,0,0.08308266599973042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,40,8,128,1,float16,fp8,0,0.08865066369374593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,40,128,1,float16,float16,0,0.05762133498986562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,40,128,1,float16,fp8,0,0.06912533442179362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,1,128,1,float16,float16,0,0.034143999218940735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,1,128,1,float16,fp8,0,0.040181333820025124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,2,128,1,float16,float16,0,0.03581333408753077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,2,128,1,float16,fp8,0,0.042090664307276406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,4,128,1,float16,float16,0,0.03863999992609024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,4,128,1,float16,fp8,0,0.045978665351867676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,8,128,1,float16,float16,0,0.03963200002908707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,40,8,128,1,float16,fp8,0,0.047135998805363975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,40,128,1,float16,float16,0,0.03245333333810171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,40,128,1,float16,fp8,0,0.037061333656311035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,1,128,1,float16,float16,0,0.02483733246723811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,1,128,1,float16,fp8,0,0.029391999046007793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,2,128,1,float16,float16,0,0.025173333783944447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,4,128,1,float16,float16,0,0.026026666164398193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,2,128,1,float16,fp8,0,0.029648000995318096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,4,128,1,float16,fp8,0,0.03196266790231069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,8,128,1,float16,float16,0,0.02611200014750163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,40,8,128,1,float16,fp8,0,0.03393599887688955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,1,128,1,float16,fp8,0,1.2915786902109783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,1,128,1,float16,float16,0,1.1099147001902263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,2,128,1,float16,fp8,0,1.3658240636189778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,2,128,1,float16,float16,0,1.2437012990315754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,4,128,1,float16,fp8,0,1.8117705980936687
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,40,128,1,float16,float16,0,1.127125342686971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,8,128,1,float16,float16,0,1.557919979095459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,40,128,1,float16,fp8,0,1.3051733175913494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,1,128,1,float16,float16,0,0.5650933186213175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,8,128,1,float16,fp8,0,1.7889013290405273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,1,128,1,float16,fp8,0,0.6619093418121338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,2,128,1,float16,float16,0,0.6336693366368612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,2,128,1,float16,fp8,0,0.6985599994659424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,4,128,1,float16,fp8,0,0.9203840096791586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,40,4,128,1,float16,float16,0,1.521173318227132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,40,128,1,float16,float16,0,0.5739200115203857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,40,128,1,float16,fp8,0,0.6665120124816895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,8,128,1,float16,float16,0,0.7928906281789144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,4,128,1,float16,float16,0,0.7704799969991049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,1,128,1,float16,float16,0,0.2940533359845479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,40,8,128,1,float16,fp8,0,0.9368693033854166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,1,128,1,float16,fp8,0,0.3419413169225057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,2,128,1,float16,float16,0,0.3407839934031169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,2,128,1,float16,fp8,0,0.3612000147501628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,4,128,1,float16,float16,0,0.3967626492182414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,8,128,1,float16,float16,0,0.4071253140767415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,4,128,1,float16,fp8,0,0.47180267175038654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,40,128,1,float16,float16,0,0.299072007338206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,2,128,1,float16,float16,0,0.1724053422609965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,40,128,1,float16,fp8,0,0.34465599060058594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,1,128,1,float16,float16,0,0.15576533476511636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,1,128,1,float16,fp8,0,0.18210132916768393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,40,8,128,1,float16,fp8,0,0.4777173201243083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,2,128,1,float16,fp8,0,0.19099199771881104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,4,128,1,float16,float16,0,0.20716800292332968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,1,128,1,float16,float16,0,0.08710400263468425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,4,128,1,float16,fp8,0,0.2455199956893921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,8,128,1,float16,float16,0,0.21224000056584677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,40,8,128,1,float16,fp8,0,0.2528533339500427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,40,128,1,float16,float16,0,0.16058133045832315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,40,128,1,float16,fp8,0,0.18522133429845175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,1,128,1,float16,fp8,0,0.10098133484522502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,2,128,1,float16,float16,0,0.09603200356165568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,2,128,1,float16,fp8,0,0.10769599676132202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,4,128,1,float16,float16,0,0.11262399951616923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,4,128,1,float16,fp8,0,0.13345600167910257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,8,128,1,float16,float16,0,0.11595199505488078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,40,8,128,1,float16,fp8,0,0.13877866665522257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,40,128,1,float16,float16,0,0.08959466218948364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,40,128,1,float16,fp8,0,0.10526933272679646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,1,128,1,float16,float16,0,0.050373335679372154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,1,128,1,float16,fp8,0,0.058730666836102806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,8,128,1,float16,fp8,0,0.07834666470686595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,2,128,1,float16,float16,0,0.05417066812515259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,2,128,1,float16,fp8,0,0.06329066554705302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,1,128,1,float16,float16,0,0.02924266705910365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,4,128,1,float16,float16,0,0.06411733229955037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,4,128,1,float16,fp8,0,0.07538666824499766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,2,128,1,float16,fp8,0,0.034661332766215004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,40,8,128,1,float16,float16,0,0.06593599915504456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,40,128,1,float16,float16,0,0.05285866558551788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,40,128,1,float16,fp8,0,0.06393600006898244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,1,128,1,float16,fp8,0,0.03435199956099192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,2,128,1,float16,float16,0,0.030293333033720653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,4,128,1,float16,float16,0,0.032885332902272545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,4,128,1,float16,fp8,0,0.040661332507928215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,8,128,1,float16,float16,0,0.03365333378314972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,1,128,1,float16,fp8,0,0.025839999318122864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,40,8,128,1,float16,fp8,0,0.04091199984153112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,40,128,1,float16,float16,0,0.02824000020821889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,40,128,1,float16,fp8,0,0.03428266694148382
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,1,128,1,float16,float16,0,0.02160533269246419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,2,128,1,float16,float16,0,0.022202665607134502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,2,128,1,float16,fp8,0,0.025594666600227356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,4,128,1,float16,float16,0,0.02310933421055476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,40,128,1,float16,fp8,0,0.022757334013779957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,4,128,1,float16,fp8,0,0.028389332195123036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,8,128,1,float16,float16,0,0.02288000037272771
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,40,8,128,1,float16,fp8,0,0.02828799933195114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,40,128,1,float16,float16,0,0.01838933303952217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,1,128,1,float16,float16,0,0.016490666816631954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,1,128,1,float16,fp8,0,0.01953599974513054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,2,128,1,float16,float16,0,0.01708799973130226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,8,128,1,float16,fp8,0,0.02063999945918719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,2,128,1,float16,fp8,0,0.019354666272799175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,4,128,1,float16,float16,0,0.016666666915019352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,4,128,1,float16,fp8,0,0.020901332298914593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,40,8,128,1,float16,float16,0,0.016800000021855038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,1,128,1,float16,float16,0,0.5405013163884481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,1,128,1,float16,fp8,0,0.5840479930241903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,2,128,1,float16,float16,0,0.6021386782328287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,4,128,1,float16,float16,0,0.7475146452585856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,4,128,1,float16,fp8,0,0.8320213158925375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,40,128,1,float16,float16,0,0.5522240002950033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,8,128,1,float16,float16,0,0.7616639931996664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,40,128,1,float16,fp8,0,0.6104480028152466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,1,128,1,float16,float16,0,0.27910399436950684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,2,128,1,float16,fp8,0,0.6260319948196411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,40,8,128,1,float16,fp8,0,0.8902506828308105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,1,128,1,float16,fp8,0,0.3052639961242676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,2,128,1,float16,fp8,0,0.3248639901479085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,2,128,1,float16,float16,0,0.3177120089530945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,4,128,1,float16,float16,0,0.3792320092519124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,4,128,1,float16,fp8,0,0.429418683052063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,8,128,1,float16,float16,0,0.39297600587209064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,40,128,1,float16,fp8,0,0.319760004679362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,40,8,128,1,float16,fp8,0,0.4530080159505208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,1,128,1,float16,float16,0,0.14989333351453146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,2,128,1,float16,fp8,0,0.17902400096257529
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,40,128,1,float16,float16,0,0.28588799635569256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,1,128,1,float16,fp8,0,0.16523200273513794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,2,128,1,float16,float16,0,0.16592533389727274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,40,128,1,float16,float16,0,0.1532906691233317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,4,128,1,float16,float16,0,0.2001919945081075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,4,128,1,float16,fp8,0,0.23092800378799438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,8,128,1,float16,float16,0,0.20569600661595663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,40,128,1,float16,fp8,0,0.17192000150680542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,2,128,1,float16,fp8,0,0.10056533416112264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,1,128,1,float16,float16,0,0.08392000198364258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,4,128,1,float16,fp8,0,0.12470933794975281
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,1,128,1,float16,fp8,0,0.09538132945696513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,40,8,128,1,float16,fp8,0,0.2380853295326233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,2,128,1,float16,float16,0,0.09241066376368205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,4,128,1,float16,float16,0,0.10894399881362915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,8,128,1,float16,fp8,0,0.1279093325138092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,2,128,1,float16,float16,0,0.05243200063705444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,40,8,128,1,float16,float16,0,0.11267733573913574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,40,128,1,float16,float16,0,0.08676266670227051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,40,128,1,float16,fp8,0,0.10019200046857198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,1,128,1,float16,float16,0,0.047007997830708824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,1,128,1,float16,fp8,0,0.05176533261934916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,2,128,1,float16,fp8,0,0.05508266886075338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,4,128,1,float16,float16,0,0.06051200131575266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,4,128,1,float16,fp8,0,0.06843199829260509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,8,128,1,float16,float16,0,0.06233066817124685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,40,8,128,1,float16,fp8,0,0.07205333312352498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,40,128,1,float16,float16,0,0.051829333106676735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,40,128,1,float16,fp8,0,0.05584533512592316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,1,128,1,float16,float16,0,0.027098665634791057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,1,128,1,float16,fp8,0,0.030261332790056866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,2,128,1,float16,float16,0,0.028016000986099243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,2,128,1,float16,fp8,0,0.031983998914559685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,4,128,1,float16,float16,0,0.030261332790056866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,4,128,1,float16,fp8,0,0.035088000198205314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,8,128,1,float16,float16,0,0.033301333586374916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,40,8,128,1,float16,fp8,0,0.035631999373435974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,40,128,1,float16,float16,0,0.028143999477227528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,4,128,1,float16,float16,0,0.022272000710169475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,40,128,1,float16,fp8,0,0.02882666637500127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,1,128,1,float16,float16,0,0.020618667205174763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,1,128,1,float16,fp8,0,0.023503998915354412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,40,128,1,float16,float16,0,0.016837333639462788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,2,128,1,float16,float16,0,0.020181333025296528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,2,128,1,float16,fp8,0,0.023365333676338196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,4,128,1,float16,fp8,0,0.025120000044504803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,8,128,1,float16,float16,0,0.022698665658632915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,40,8,128,1,float16,fp8,0,0.025285333395004272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,40,128,1,float16,fp8,0,0.018816000471512478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,1,128,1,float16,float16,0,0.01532799998919169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,8,128,1,float16,float16,0,0.015386667102575302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,1,128,1,float16,fp8,0,0.017781333376963932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,2,128,1,float16,float16,0,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,2,128,1,float16,fp8,0,0.017386666188637417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,4,128,1,float16,float16,0,0.015279999623696009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,4,128,1,float16,fp8,0,0.01748266691962878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,1,128,1,float16,fp8,0,0.017071999609470367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,40,8,128,1,float16,fp8,0,0.017632000148296356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,40,128,1,float16,float16,0,0.015861333658297855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,40,128,1,float16,fp8,0,0.018309333672126133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,1,128,1,float16,float16,0,0.014544000228246054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,2,128,1,float16,float16,0,0.014991999914248785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,8,128,1,float16,fp8,0,0.017237332959969837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,2,128,1,float16,fp8,0,0.01708799973130226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,4,128,1,float16,float16,0,0.014720000326633453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,4,128,1,float16,fp8,0,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,40,8,128,1,float16,float16,0,0.014826666563749313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,1,128,1,float16,fp8,0,0.30557332436243695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,1,128,1,float16,float16,0,0.27830400069554645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,2,128,1,float16,float16,0,0.31312533219655353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,40,128,1,float16,float16,0,0.3171786665916443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,40,128,1,float16,fp8,0,0.31670933961868286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,4,128,1,float16,fp8,0,0.4339199860890706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,4,128,1,float16,float16,0,0.38356268405914307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,2,128,1,float16,fp8,0,0.3257546623547872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,1,128,1,float16,float16,0,0.1497119963169098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,8,128,1,float16,float16,0,0.4266293446222941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,40,8,128,1,float16,fp8,0,0.4708266655604045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,1,128,1,float16,fp8,0,0.16528000434239706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,2,128,1,float16,fp8,0,0.17683732509613037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,4,128,1,float16,fp8,0,0.22984000047047934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,8,128,1,float16,float16,0,0.22126400470733643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,2,128,1,float16,float16,0,0.16613866885503134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,4,128,1,float16,float16,0,0.2013173302014669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,40,128,1,float16,float16,0,0.16686399777730307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,1,128,1,float16,float16,0,0.08534399668375652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,40,8,128,1,float16,fp8,0,0.25252799193064374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,1,128,1,float16,fp8,0,0.09434133768081665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,40,128,1,float16,fp8,0,0.1692053278287252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,8,128,1,float16,float16,0,0.11873066425323486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,2,128,1,float16,float16,0,0.09340799848238628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,2,128,1,float16,fp8,0,0.10140267014503479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,4,128,1,float16,float16,0,0.1092639962832133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,40,128,1,float16,fp8,0,0.09237333138783772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,1,128,1,float16,float16,0,0.04882133503754934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,4,128,1,float16,fp8,0,0.12461866935094197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,40,8,128,1,float16,fp8,0,0.1378666659196218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,40,128,1,float16,float16,0,0.09172266721725464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,1,128,1,float16,fp8,0,0.04951466619968414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,2,128,1,float16,float16,0,0.05161599814891815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,8,128,1,float16,fp8,0,0.07577066620190938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,2,128,1,float16,fp8,0,0.053216000398000084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,4,128,1,float16,float16,0,0.05964266757170359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,4,128,1,float16,fp8,0,0.06749866902828217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,40,8,128,1,float16,float16,0,0.06763199965159099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,40,128,1,float16,float16,0,0.05272000034650167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,40,128,1,float16,fp8,0,0.048938666780789696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,1,128,1,float16,float16,0,0.026005332668622334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,1,128,1,float16,fp8,0,0.030250666042168934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,2,128,1,float16,float16,0,0.027322667340437572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,2,128,1,float16,fp8,0,0.031632001201311745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,4,128,1,float16,float16,0,0.03057066599527995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,4,128,1,float16,fp8,0,0.035045333206653595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,8,128,1,float16,fp8,0,0.035562666753927864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,2,128,1,float16,float16,0,0.020554666717847187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,2,128,1,float16,fp8,0,0.023562667270501454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,40,8,128,1,float16,float16,0,0.03160533308982849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,40,128,1,float16,float16,0,0.027130665878454845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,8,128,1,float16,float16,0,0.02182399978240331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,40,128,1,float16,fp8,0,0.024911999702453613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,1,128,1,float16,float16,0,0.01989866668979327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,40,128,1,float16,fp8,0,0.01926933353145917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,1,128,1,float16,fp8,0,0.023525332411130268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,4,128,1,float16,float16,0,0.021770666042963665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,2,128,1,float16,float16,0,0.015274666249752045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,4,128,1,float16,fp8,0,0.025098666548728943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,4,128,1,float16,float16,0,0.015610666324694952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,40,8,128,1,float16,fp8,0,0.02497066557407379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,8,128,1,float16,float16,0,0.015439999600251516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,40,128,1,float16,float16,0,0.018016000588734944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,1,128,1,float16,float16,0,0.014943999548753103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,40,128,1,float16,float16,0,0.013663999736309052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,1,128,1,float16,fp8,0,0.01727466657757759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,2,128,1,float16,fp8,0,0.017386666188637417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,4,128,1,float16,fp8,0,0.01735466718673706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,2,128,1,float16,fp8,0,0.017162666966517765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,40,8,128,1,float16,fp8,0,0.017781333376963932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,40,128,1,float16,fp8,0,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,1,128,1,float16,fp8,0,0.017557332913080852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,1,128,1,float16,float16,0,0.01458666721979777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,40,128,1,float16,float16,0,0.013493333011865616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,2,128,1,float16,float16,0,0.014826666563749313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,4,128,1,float16,float16,0,0.015050667027632395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,1,128,1,float16,fp8,0,0.016303999970356624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,4,128,1,float16,fp8,0,0.017466666797796886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,2,128,1,float16,fp8,0,0.017018667111794155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,8,128,1,float16,float16,0,0.01452800010641416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,40,8,128,1,float16,fp8,0,0.017152000218629837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,40,128,1,float16,fp8,0,0.014309333016475042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,8,128,1,float16,fp8,0,0.016783999900023144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,1,128,1,float16,float16,0,0.013989333063364029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,2,128,1,float16,float16,0,0.01421333352724711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,4,128,1,float16,float16,0,0.01434133326013883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,4,128,1,float16,fp8,0,0.016314666718244553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,40,8,128,1,float16,float16,0,0.014544000228246054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,1,128,1,float16,float16,0,0.14941333731015524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,1,128,1,float16,fp8,0,0.16477866967519125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,2,128,1,float16,float16,0,0.16524266203244528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,2,128,1,float16,fp8,0,0.17926400899887085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,4,128,1,float16,float16,0,0.21398399273554483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,4,128,1,float16,fp8,0,0.24597332874933878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,40,128,1,float16,float16,0,0.22885332504908243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,8,128,1,float16,float16,0,0.24843732515970865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,40,128,1,float16,fp8,0,0.22501333554585776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,40,8,128,1,float16,fp8,0,0.2903626759847005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,1,128,1,float16,float16,0,0.0844533344109853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,1,128,1,float16,fp8,0,0.09504533807436626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,2,128,1,float16,float16,0,0.09268800417582194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,2,128,1,float16,fp8,0,0.10259200135866801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,4,128,1,float16,float16,0,0.11473600069681804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,4,128,1,float16,fp8,0,0.1356000006198883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,8,128,1,float16,fp8,0,0.15784533818562826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,40,8,128,1,float16,float16,0,0.13301866253217062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,40,128,1,float16,float16,0,0.12198932965596516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,40,128,1,float16,fp8,0,0.1195146640141805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,2,128,1,float16,fp8,0,0.05469333132108053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,1,128,1,float16,float16,0,0.047584002216657005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,1,128,1,float16,fp8,0,0.04990399877230326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,2,128,1,float16,float16,0,0.05156800150871277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,4,128,1,float16,float16,0,0.06515733400980632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,4,128,1,float16,fp8,0,0.0735040009021759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,8,128,1,float16,float16,0,0.07524266839027405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,40,8,128,1,float16,fp8,0,0.08769599596659343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,40,128,1,float16,fp8,0,0.06462400158246358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,40,128,1,float16,float16,0,0.06744533280531566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,1,128,1,float16,fp8,0,0.029994666576385498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,1,128,1,float16,float16,0,0.025914666553338368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,8,128,1,float16,float16,0,0.03621333340803782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,2,128,1,float16,float16,0,0.02752000093460083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,8,128,1,float16,fp8,0,0.04084266722202301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,2,128,1,float16,fp8,0,0.03175999969244003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,4,128,1,float16,float16,0,0.030000001192092896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,40,4,128,1,float16,fp8,0,0.03510399907827377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,40,128,1,float16,float16,0,0.032831999162832894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,40,128,1,float16,fp8,0,0.03160000095764796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,1,128,1,float16,float16,0,0.019738666713237762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,1,128,1,float16,fp8,0,0.023376000424226124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,2,128,1,float16,float16,0,0.020714666694402695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,2,128,1,float16,fp8,0,0.023503998915354412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,4,128,1,float16,float16,0,0.021749332547187805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,4,128,1,float16,fp8,0,0.025008000433444977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,8,128,1,float16,float16,0,0.021274665991465252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,40,8,128,1,float16,fp8,0,0.025055999557177227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,40,128,1,float16,float16,0,0.020746666938066483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,40,128,1,float16,fp8,0,0.02124800036350886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,1,128,1,float16,float16,0,0.015365333606799444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,1,128,1,float16,fp8,0,0.017765333255132038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,2,128,1,float16,float16,0,0.014906667172908783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,2,128,1,float16,fp8,0,0.017632000148296356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,40,128,1,float16,fp8,0,0.01764800027012825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,4,128,1,float16,float16,0,0.015354666858911514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,4,128,1,float16,fp8,0,0.01766933376590411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,8,128,1,float16,float16,0,0.014864000181357065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,40,8,128,1,float16,fp8,0,0.01741333305835724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,40,128,1,float16,float16,0,0.016623999923467636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,1,128,1,float16,float16,0,0.014661333213249842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,1,128,1,float16,fp8,0,0.017658667018016178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,2,128,1,float16,float16,0,0.015119999647140503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,40,128,1,float16,float16,0,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,2,128,1,float16,fp8,0,0.0173333336909612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,4,128,1,float16,float16,0,0.014394666999578476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,1,128,1,float16,fp8,0,0.016885332763195038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,4,128,1,float16,fp8,0,0.017077332983414333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,8,128,1,float16,float16,0,0.0144213338692983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,4,128,1,float16,float16,0,0.014474666366974512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,40,8,128,1,float16,fp8,0,0.017338667064905167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,40,128,1,float16,fp8,0,0.014432000617186228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,1,128,1,float16,float16,0,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,2,128,1,float16,float16,0,0.014101333916187286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,2,128,1,float16,fp8,0,0.016336000214020412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,4,128,1,float16,fp8,0,0.01685333376129468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,8,128,1,float16,float16,0,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,40,8,128,1,float16,fp8,0,0.016672000288963318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,2,128,1,float16,fp8,0,0.015578666081031164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,40,128,1,float16,float16,0,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,40,128,1,float16,fp8,0,0.014405333747466406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,1,128,1,float16,float16,0,0.013738666971524557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,1,128,1,float16,fp8,0,0.01552533358335495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,2,128,1,float16,float16,0,0.013674666484196981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,4,128,1,float16,float16,0,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,4,128,1,float16,fp8,0,0.01602666700879733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,8,128,1,float16,float16,0,0.013776000589132309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,40,8,128,1,float16,fp8,0,0.015813333292802174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,1,128,1,float16,float16,0,0.08455466230710347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,1,128,1,float16,fp8,0,0.09548266728719075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,2,128,1,float16,fp8,0,0.11116266250610352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,2,128,1,float16,float16,0,0.09855467081069946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,4,128,1,float16,float16,0,0.12930666406949362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,4,128,1,float16,fp8,0,0.154858668645223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,8,128,1,float16,float16,0,0.13763200243314108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,40,128,1,float16,float16,0,0.18623467286427817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,40,128,1,float16,fp8,0,0.18279999494552612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,40,8,128,1,float16,fp8,0,0.1588479975859324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,1,128,1,float16,float16,0,0.04797866443792979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,1,128,1,float16,fp8,0,0.05123733480771383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,2,128,1,float16,float16,0,0.05634133517742157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,2,128,1,float16,fp8,0,0.059749335050582886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,4,128,1,float16,float16,0,0.07258666555086772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,4,128,1,float16,fp8,0,0.08392533659934998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,8,128,1,float16,float16,0,0.07733866572380066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,40,8,128,1,float16,fp8,0,0.0869653324286143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,40,128,1,float16,float16,0,0.09898133079210918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,40,128,1,float16,fp8,0,0.09627733627955119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,1,128,1,float16,float16,0,0.026367999613285065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,4,128,1,float16,float16,0,0.03443199892838796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,1,128,1,float16,fp8,0,0.030031998952229817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,4,128,1,float16,fp8,0,0.04066666712363561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,2,128,1,float16,float16,0,0.027535999814669292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,2,128,1,float16,fp8,0,0.03139200061559677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,8,128,1,float16,float16,0,0.03540800015131632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,40,8,128,1,float16,fp8,0,0.04030400017897288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,40,128,1,float16,float16,0,0.04914666712284088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,40,128,1,float16,fp8,0,0.04690133531888326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,1,128,1,float16,float16,0,0.0198186660806338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,1,128,1,float16,fp8,0,0.023562667270501454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,2,128,1,float16,float16,0,0.020175999651352566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,4,128,1,float16,fp8,0,0.025258667767047882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,2,128,1,float16,fp8,0,0.023376000424226124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,4,128,1,float16,float16,0,0.021418665846188862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,8,128,1,float16,float16,0,0.021269333859284718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,40,8,128,1,float16,fp8,0,0.0249439999461174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,40,128,1,float16,float16,0,0.02773333340883255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,1,128,1,float16,float16,0,0.015141333142916361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,40,128,1,float16,fp8,0,0.02787200113137563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,1,128,1,float16,fp8,0,0.017840000490347546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,8,128,1,float16,float16,0,0.015034666905800501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,2,128,1,float16,float16,0,0.015072000523408255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,40,128,1,float16,float16,0,0.019989332805077236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,2,128,1,float16,fp8,0,0.017322666943073273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,4,128,1,float16,float16,0,0.014912000546852747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,4,128,1,float16,fp8,0,0.01724799970785777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,40,8,128,1,float16,fp8,0,0.017344000438849132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,40,128,1,float16,fp8,0,0.020768000433842342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,1,128,1,float16,float16,0,0.014831999937693277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,1,128,1,float16,fp8,0,0.017466666797796886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,2,128,1,float16,float16,0,0.014970666418472925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,2,128,1,float16,fp8,0,0.017344000438849132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,4,128,1,float16,float16,0,0.014725333700577417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,1,128,1,float16,float16,0,0.014197333405415217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,4,128,1,float16,fp8,0,0.016949333250522614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,8,128,1,float16,float16,0,0.01444799949725469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,40,8,128,1,float16,fp8,0,0.017157333592573803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,4,128,1,float16,float16,0,0.014362666755914688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,40,128,1,float16,float16,0,0.0162773331006368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,40,128,1,float16,fp8,0,0.0174346665541331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,1,128,1,float16,fp8,0,0.016229332735141117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,2,128,1,float16,float16,0,0.0143306665122509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,2,128,1,float16,fp8,0,0.016757333030303318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,4,128,1,float16,fp8,0,0.01685333376129468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,8,128,1,float16,float16,0,0.01440000037352244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,40,8,128,1,float16,fp8,0,0.01655999943614006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,40,128,1,float16,float16,0,0.013306666165590286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,40,128,1,float16,fp8,0,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,1,128,1,float16,float16,0,0.013663999736309052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,8,128,1,float16,float16,0,0.013557333499193192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,1,128,1,float16,fp8,0,0.015669333438078564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,2,128,1,float16,float16,0,0.013839999834696451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,40,128,1,float16,fp8,0,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,2,128,1,float16,fp8,0,0.015824000040690105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,4,128,1,float16,float16,0,0.013728000223636627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,4,128,1,float16,fp8,0,0.01647466669480006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,40,8,128,1,float16,fp8,0,0.01569066693385442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,40,128,1,float16,float16,0,0.012874666601419449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,4,128,1,float16,fp8,0,0.015493333339691162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,1,128,1,float16,float16,0,0.013610667238632837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,1,128,1,float16,fp8,0,0.015541333705186844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,2,128,1,float16,float16,0,0.013610667238632837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,2,128,1,float16,fp8,0,0.015562667200962702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,4,128,1,float16,float16,0,0.013562666873137156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,8,128,1,float16,float16,0,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,40,8,128,1,float16,fp8,0,0.015461333096027374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,0,0.02183466653029124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,0,0.023631999890009563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,0,0.028229333460330963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,0,0.03035733352104823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,0,0.04135466615358988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,0,0.04387733340263367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,0,0.06655466556549072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,0,0.07066133121649425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,40,128,1,float16,float16,0,0.14049599568049112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,40,128,1,float16,fp8,0,0.14822399616241455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,0,0.01929066702723503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,0,0.014655999839305878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,0,0.016058667252461117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,0,0.01844800015290578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,0,0.02477866659561793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,0,0.026613332331180573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,0,0.037765334049860634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,0,0.04068800061941147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,40,128,1,float16,fp8,0,0.08046400050322215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,0,0.014080000420411428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,40,128,1,float16,float16,0,0.07515199979146321
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,0,0.012837332983811697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,0,0.02362666775782903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,0,0.013477332890033722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,0,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,0,0.016714667280515034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,0,0.01815466706951459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,0,0.025242666403452556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,40,128,1,float16,float16,0,0.0439626673857371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,40,128,1,float16,fp8,0,0.0466186652580897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,0,0.012432000289360682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,0,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,0,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,0,0.013194666554530462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,0,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,0,0.013381333400805792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,0,0.01647466669480006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,0,0.0176959993938605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,40,128,1,float16,float16,0,0.026741333305835724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,40,128,1,float16,fp8,0,0.029440000653266907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,0,0.011653333902359009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,0,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,0,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,0,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,0,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,0,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,0,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,0,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,0,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,40,128,1,float16,float16,0,0.019637333850065868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,40,128,1,float16,fp8,0,0.021514666577180225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,0,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,0,0.012719999998807907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,0,0.011472000430027643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,0,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,0,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,0,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,0,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,0,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,40,128,1,float16,float16,0,0.016069332758585613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,0,0.010922666639089584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,40,128,1,float16,fp8,0,0.017616000026464462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,0,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,40,128,1,float16,float16,0,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,0,0.011685332904259363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,0,0.011429333438475927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,0,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,0,0.011541333049535751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,0,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,40,128,1,float16,fp8,0,0.014186666657527288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,0,0.011077333241701126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,0,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,0,0.011114666859308878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,0,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,0,0.011477333803971609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,0,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,0,0.011744000017642975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,0,0.012506666282812754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,40,128,1,float16,float16,0,0.012506666282812754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,0,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,40,128,1,float16,fp8,0,0.014266667266686758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,0,0.01098666712641716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,0,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,0,0.011381333072980246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,0,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,0,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,0,0.011600000162919363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,0,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,1,128,1,float16,fp8,0,27.243860880533855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,1,128,1,float16,float16,0,30.817433675130207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,2,128,1,float16,float16,0,30.48632558186849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,4,128,1,float16,fp8,0,28.34534962972005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,8,128,1,float16,fp8,0,28.063669840494793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,4,128,1,float16,float16,0,30.183977762858074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,8,128,1,float16,float16,0,31.32341766357422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,32,128,1,float16,float16,0,16.635237375895183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,32,128,1,float16,fp8,0,14.5970458984375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,1,128,1,float16,float16,0,15.085163116455078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,1,128,1,float16,fp8,0,13.72531763712565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,2,128,1,float16,fp8,0,13.780906677246094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,2,128,1,float16,float16,0,14.823023478190104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,4,128,1,float16,float16,0,15.033551534016928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,32,2,128,1,float16,fp8,0,55.97588094075521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,32,128,1,float16,fp8,0,7.3584855397542315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,32,128,1,float16,float16,0,8.433546702067057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,1,128,1,float16,float16,0,7.753898620605469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,1,128,1,float16,fp8,0,6.923493067423503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,4,128,1,float16,fp8,0,14.259583791097006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,8,128,1,float16,fp8,0,14.188624064127604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,32,8,128,1,float16,float16,0,15.873887379964193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,2,128,1,float16,fp8,0,6.951770782470703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,4,128,1,float16,float16,0,7.488202412923177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,4,128,1,float16,fp8,0,7.254271825154622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,8,128,1,float16,float16,0,8.240287780761719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,32,128,1,float16,float16,0,3.822271982828776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,32,128,1,float16,fp8,0,4.001765251159668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,8,128,1,float16,fp8,0,7.115781148274739
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,1,128,1,float16,float16,0,3.6766506830851235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,2,128,1,float16,float16,0,3.592303911844889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,1,128,1,float16,fp8,0,4.085760116577148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,2,128,1,float16,fp8,0,3.733440081278483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,4,128,1,float16,float16,0,4.631706555684407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,4,128,1,float16,fp8,0,3.7062721252441406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,8,128,1,float16,float16,0,3.792970657348633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,32,2,128,1,float16,float16,0,21.972757975260418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,32,8,128,1,float16,fp8,0,3.859583854675293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,1,128,1,float16,float16,0,18.104843139648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,1,128,1,float16,fp8,0,15.86312484741211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,2,128,1,float16,fp8,0,15.825983683268229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,2,128,1,float16,float16,0,17.224112192789715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,4,128,1,float16,float16,0,18.093748728434246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,8,128,1,float16,float16,0,18.302608489990234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,32,128,1,float16,float16,0,9.291557312011719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,1,128,1,float16,float16,0,8.58737055460612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,32,128,1,float16,fp8,0,8.55400021870931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,1,128,1,float16,fp8,0,7.889743804931641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,8,128,1,float16,fp8,0,16.39196268717448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,2,128,1,float16,float16,0,8.802938461303711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,2,128,1,float16,fp8,0,7.985573450724284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,32,4,128,1,float16,fp8,0,28.299359639485676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,32,128,1,float16,float16,0,4.5272267659505205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,32,128,1,float16,fp8,0,4.592831929524739
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,1,128,1,float16,float16,0,4.170447985331218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,4,128,1,float16,float16,0,8.492005030314127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,4,128,1,float16,fp8,0,8.365392049153646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,8,128,1,float16,float16,0,8.82692782084147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,32,8,128,1,float16,fp8,0,8.26042111714681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,1,128,1,float16,fp8,0,7.3167572021484375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,2,128,1,float16,float16,0,4.310442606608073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,4,128,1,float16,float16,0,4.080522537231445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,2,128,1,float16,fp8,0,4.095402717590332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,32,128,1,float16,float16,0,2.239370663960775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,4,128,1,float16,fp8,0,4.593125343322754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,8,128,1,float16,float16,0,4.215466817220052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,32,8,128,1,float16,fp8,0,4.615770657857259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,1,128,1,float16,float16,0,2.071061293284098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,32,128,1,float16,fp8,0,3.2418187459309897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,1,128,1,float16,fp8,0,2.330165386199951
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,2,128,1,float16,float16,0,2.058736006418864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,2,128,1,float16,fp8,0,2.0758132934570312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,4,128,1,float16,float16,0,2.061914602915446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,4,128,1,float16,fp8,0,2.2540693283081055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,8,128,1,float16,float16,0,2.147221406300863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,32,8,128,1,float16,fp8,0,2.1891627311706543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,1,128,1,float16,fp8,0,11.09548314412435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,2,128,1,float16,fp8,0,11.169989267985025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,2,128,1,float16,float16,0,12.652325948079428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,4,128,1,float16,float16,0,12.606309254964193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,4,128,1,float16,fp8,0,11.67673110961914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,8,128,1,float16,fp8,0,11.633883158365885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,8,128,1,float16,float16,0,12.99820327758789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,32,128,1,float16,float16,0,6.726165135701497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,32,128,1,float16,fp8,0,6.110160191853841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,1,128,1,float16,float16,0,5.860928217569987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,1,128,1,float16,fp8,0,5.654336293538411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,2,128,1,float16,float16,0,6.404117584228516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,2,128,1,float16,fp8,0,5.601151784261067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,32,1,128,1,float16,float16,0,26.599685668945312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,4,128,1,float16,float16,0,5.974357604980469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,32,128,1,float16,float16,0,3.196394602457682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,4,128,1,float16,fp8,0,6.239615758260091
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,1,128,1,float16,float16,0,2.8376213709513345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,32,128,1,float16,fp8,0,3.3457066218058267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,8,128,1,float16,float16,0,6.393781026204427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,2,128,1,float16,float16,0,2.9620800018310547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,32,8,128,1,float16,fp8,0,6.277983983357747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,1,128,1,float16,fp8,0,5.09226671854655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,2,128,1,float16,fp8,0,3.020240147908529
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,4,128,1,float16,float16,0,2.922634760538737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,4,128,1,float16,fp8,0,2.997525215148926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,32,128,1,float16,float16,0,1.6180052757263184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,8,128,1,float16,float16,0,2.9787092208862305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,32,128,1,float16,fp8,0,1.641818682352702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,32,8,128,1,float16,fp8,0,3.0944693883260093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,1,128,1,float16,fp8,0,1.500490665435791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,2,128,1,float16,float16,0,1.501530647277832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,1,128,1,float16,float16,0,3.704373359680176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,2,128,1,float16,fp8,0,1.462922732035319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,8,128,1,float16,fp8,0,1.5434506734212239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,4,128,1,float16,float16,0,1.4895040194193523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,4,128,1,float16,fp8,0,1.608565330505371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,32,8,128,1,float16,float16,0,1.5120213826497395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,1,128,1,float16,fp8,0,14.427188873291016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,1,128,1,float16,float16,0,16.013460795084637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,2,128,1,float16,fp8,0,14.587540944417318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,4,128,1,float16,float16,0,15.983109792073568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,4,128,1,float16,fp8,0,15.459376017252604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,8,128,1,float16,fp8,0,15.282816569010416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,8,128,1,float16,float16,0,16.829327901204426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,1,128,1,float16,float16,0,7.7346986134847
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,32,128,1,float16,float16,0,9.191738764444986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,32,128,1,float16,fp8,0,8.114320119222006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,1,128,1,float16,fp8,0,7.2419789632161455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,2,128,1,float16,float16,0,8.278645197550455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,2,128,1,float16,fp8,0,7.291082382202148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,4,128,1,float16,float16,0,8.305023829142252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,32,128,1,float16,float16,0,4.12552007039388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,1,128,1,float16,float16,0,3.6419947942097983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,32,128,1,float16,fp8,0,4.313808123270671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,1,128,1,float16,fp8,0,3.796527862548828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,4,128,1,float16,fp8,0,7.922400156656901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,8,128,1,float16,fp8,0,7.672789255777995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,32,8,128,1,float16,float16,0,8.512016296386719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,2,128,1,float16,float16,0,3.967205365498861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,2,128,1,float16,fp8,0,3.932954788208008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,4,128,1,float16,float16,0,3.912912050882975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,4,128,1,float16,fp8,0,3.8930934270222983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,8,128,1,float16,float16,0,4.082223892211914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,32,128,1,float16,float16,0,2.139557361602783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,32,8,128,1,float16,fp8,0,4.0846506754557295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,32,128,1,float16,fp8,0,2.1896106402079263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,1,128,1,float16,float16,0,1.9879520734151204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,1,128,1,float16,fp8,0,1.857034683227539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,32,2,128,1,float16,float16,0,43.45575968424479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,2,128,1,float16,float16,0,2.0507787068684897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,2,128,1,float16,fp8,0,1.9733920097351074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,4,128,1,float16,float16,0,1.9143625895182292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,4,128,1,float16,fp8,0,1.9895413716634114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,8,128,1,float16,float16,0,1.9886293411254883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,32,128,1,float16,float16,0,1.2647093137105305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,32,8,128,1,float16,fp8,0,1.9724799791971843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,32,128,1,float16,fp8,0,1.1587413152058919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,1,128,1,float16,float16,0,0.963327964146932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,2,128,1,float16,float16,0,0.9744213422139486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,1,128,1,float16,fp8,0,0.9705599943796793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,2,128,1,float16,fp8,0,0.9814026355743408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,4,128,1,float16,float16,0,0.9915893077850342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,4,128,1,float16,fp8,0,1.4052054087320964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,8,128,1,float16,float16,0,1.0068373680114746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,32,8,128,1,float16,fp8,0,1.0634559790293376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,1,128,1,float16,float16,0,9.237685521443685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,1,128,1,float16,fp8,0,8.407183965047201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,2,128,1,float16,fp8,0,8.499594370524088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,4,128,1,float16,float16,0,9.570800145467123
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,2,128,1,float16,float16,0,9.286624272664389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,8,128,1,float16,float16,0,10.298730850219727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,8,128,1,float16,fp8,0,9.154325485229492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,32,128,1,float16,float16,0,5.006741205851237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,32,128,1,float16,fp8,0,4.859525362650554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,1,128,1,float16,float16,0,4.245973269144694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,1,128,1,float16,fp8,0,4.741344134012858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,2,128,1,float16,float16,0,4.58296012878418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,2,128,1,float16,fp8,0,4.297381401062012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,4,128,1,float16,float16,0,4.562416076660156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,32,128,1,float16,float16,0,2.46888001759847
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,32,4,128,1,float16,fp8,0,19.79719416300456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,4,128,1,float16,fp8,0,4.825898806254069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,8,128,1,float16,float16,0,4.7455094655354815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,32,128,1,float16,fp8,0,2.5586986541748047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,32,8,128,1,float16,fp8,0,4.768032073974609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,1,128,1,float16,float16,0,2.1233867009480796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,1,128,1,float16,fp8,0,2.283343950907389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,2,128,1,float16,float16,0,2.197727998097738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,4,128,1,float16,float16,0,2.2441760698954263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,4,128,1,float16,fp8,0,2.4902772903442383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,32,128,1,float16,float16,0,1.2589493592580159
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,8,128,1,float16,float16,0,2.290832042694092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,8,128,1,float16,fp8,0,2.3368213971455893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,32,128,1,float16,fp8,0,1.3180480003356934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,1,128,1,float16,float16,0,1.0946773687998455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,32,2,128,1,float16,fp8,0,5.087264060974121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,2,128,1,float16,fp8,0,1.1339413324991863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,1,128,1,float16,fp8,0,1.177850643793742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,2,128,1,float16,float16,0,1.1269013086954753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,4,128,1,float16,float16,0,1.135690689086914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,32,128,1,float16,fp8,0,0.6876586278279623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,32,128,1,float16,float16,0,0.9800960222880045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,4,128,1,float16,fp8,0,1.1968053181966145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,8,128,1,float16,float16,0,1.174554665883382
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,32,8,128,1,float16,fp8,0,1.2247947057088215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,1,128,1,float16,float16,0,0.5825759967168173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,1,128,1,float16,fp8,0,0.59825066725413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,2,128,1,float16,float16,0,0.5776480038960775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,2,128,1,float16,fp8,0,0.6040319999059042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,4,128,1,float16,float16,0,0.7143733501434326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,4,128,1,float16,fp8,0,0.6266613403956095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,8,128,1,float16,float16,0,0.6107199986775717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,32,8,128,1,float16,fp8,0,0.6500639915466309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,1,128,1,float16,float16,0,8.635509490966797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,2,128,1,float16,float16,0,8.715279897054037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,2,128,1,float16,fp8,0,8.236757278442383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,1,128,1,float16,fp8,0,8.032896041870117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,4,128,1,float16,fp8,0,8.861573537190756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,8,128,1,float16,float16,0,9.512191772460938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,8,128,1,float16,fp8,0,8.860879898071289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,32,128,1,float16,float16,0,4.866117477416992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,1,128,1,float16,float16,0,3.9269011815389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,32,128,1,float16,fp8,0,5.05514653523763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,1,128,1,float16,fp8,0,4.005114555358887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,2,128,1,float16,float16,0,4.134794553120931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,2,128,1,float16,fp8,0,4.241648038228353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,4,128,1,float16,float16,0,4.341834704081218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,32,128,1,float16,float16,0,2.4225120544433594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,32,128,1,float16,fp8,0,2.6617600123087564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,1,128,1,float16,float16,0,1.988767941792806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,8,128,1,float16,float16,0,4.4999574025472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,4,128,1,float16,fp8,0,4.7629439036051435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,1,128,1,float16,fp8,0,2.2148000399271646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,32,8,128,1,float16,fp8,0,4.451855977376302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,32,4,128,1,float16,float16,0,22.955482482910156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,2,128,1,float16,float16,0,2.0737226804097495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,2,128,1,float16,fp8,0,2.1269280115763345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,4,128,1,float16,float16,0,2.1389759381612143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,4,128,1,float16,fp8,0,2.373861312866211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,32,128,1,float16,float16,0,1.2189599672953289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,8,128,1,float16,float16,0,2.2302346229553223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,32,8,128,1,float16,fp8,0,2.380234718322754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,1,128,1,float16,float16,0,1.0004479885101318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,32,128,1,float16,fp8,0,1.3148106733957927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,1,128,1,float16,fp8,0,1.0564320087432861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,2,128,1,float16,float16,0,1.0465386708577473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,2,128,1,float16,fp8,0,1.0614026387532551
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,4,128,1,float16,float16,0,1.087498664855957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,8,128,1,float16,fp8,0,1.216037352879842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,4,128,1,float16,fp8,0,1.17413330078125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,32,8,128,1,float16,float16,0,1.1265066464742024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,32,128,1,float16,float16,0,0.6335839827855428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,32,128,1,float16,fp8,0,0.6704213619232178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,1,128,1,float16,float16,0,0.5136106808980306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,1,128,1,float16,fp8,0,0.5550826787948608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,2,128,1,float16,float16,0,0.5426559845606486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,2,128,1,float16,fp8,0,0.55566934744517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,4,128,1,float16,float16,0,0.5569973389307658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,4,128,1,float16,fp8,0,0.6091733376185099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,8,128,1,float16,float16,0,0.5844693183898926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,32,8,128,1,float16,fp8,0,0.6122080087661743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,32,128,1,float16,float16,0,0.34142935276031494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,32,128,1,float16,fp8,0,0.36264534791310626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,1,128,1,float16,float16,0,0.27804267406463623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,1,128,1,float16,fp8,0,0.29966400067011517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,2,128,1,float16,fp8,0,0.30688534180323285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,2,128,1,float16,float16,0,0.29628799359003705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,4,128,1,float16,float16,0,0.3001333276430766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,4,128,1,float16,fp8,0,0.31867732604344684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,8,128,1,float16,float16,0,0.31773332754770917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,32,8,128,1,float16,fp8,0,0.33716801802317303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,1,128,1,float16,float16,0,4.942117373148601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,1,128,1,float16,fp8,0,4.8738454182942705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,2,128,1,float16,float16,0,5.007280031840007
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,4,128,1,float16,float16,0,5.239386558532715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,4,128,1,float16,fp8,0,5.444490432739258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,8,128,1,float16,float16,0,5.4038130442301435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,32,128,1,float16,float16,0,2.9842828114827475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,8,128,1,float16,fp8,0,5.606735865275065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,1,128,1,float16,float16,0,2.571194648742676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,32,128,1,float16,fp8,0,3.0707947413126626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,32,2,128,1,float16,fp8,0,8.802229563395182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,1,128,1,float16,fp8,0,2.6035946210225425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,2,128,1,float16,float16,0,2.420090675354004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,2,128,1,float16,fp8,0,2.5636746088663735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,4,128,1,float16,float16,0,2.592101256052653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,4,128,1,float16,fp8,0,2.73468812306722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,8,128,1,float16,float16,0,2.6959520975748696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,32,128,1,float16,fp8,0,1.5630186398824055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,32,8,128,1,float16,fp8,0,2.7666613260904946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,1,128,1,float16,float16,0,1.1763412952423096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,1,128,1,float16,fp8,0,1.2296213308970134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,32,128,1,float16,float16,0,6.889589309692383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,2,128,1,float16,float16,0,1.2414720058441162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,2,128,1,float16,fp8,0,1.2579572995503743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,4,128,1,float16,float16,0,1.3040906588236492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,4,128,1,float16,fp8,0,1.3931093215942383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,8,128,1,float16,float16,0,1.3351200421651204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,32,8,128,1,float16,fp8,0,1.425439993540446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,1,128,1,float16,fp8,0,0.6331040064493815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,32,128,1,float16,float16,0,0.7701226870218912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,1,128,1,float16,float16,0,0.6053226788838705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,32,128,1,float16,fp8,0,2.2813119888305664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,2,128,1,float16,float16,0,0.6311306556065878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,2,128,1,float16,fp8,0,0.6501493453979492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,4,128,1,float16,float16,0,0.662714680035909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,4,128,1,float16,fp8,0,0.7289226849873861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,8,128,1,float16,float16,0,0.6930560270945231
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,32,128,1,float16,float16,0,0.40458134810129803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,32,8,128,1,float16,fp8,0,0.7365547021230062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,1,128,1,float16,float16,0,0.3185386657714844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,32,128,1,float16,fp8,0,1.9820373853047688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,1,128,1,float16,fp8,0,0.33879999319712323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,2,128,1,float16,float16,0,0.33129066228866577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,32,128,1,float16,fp8,0,0.23738133907318115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,2,128,1,float16,fp8,0,0.349018653233846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,1,128,1,float16,float16,0,0.17319466670354208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,4,128,1,float16,float16,0,0.3503146568934123
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,4,128,1,float16,fp8,0,0.3877653280893962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,8,128,1,float16,float16,0,0.3722453514734904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,32,8,128,1,float16,fp8,0,0.3973653316497803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,32,128,1,float16,float16,0,0.45316267013549805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,1,128,1,float16,fp8,0,0.18857600291570029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,2,128,1,float16,float16,0,0.18145066499710083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,2,128,1,float16,fp8,0,0.1958720088005066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,4,128,1,float16,float16,0,0.19022399187088013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,4,128,1,float16,fp8,0,0.21065600713094076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,8,128,1,float16,float16,0,0.2030293345451355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,32,8,128,1,float16,fp8,0,0.22012799978256226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,1,128,1,float16,fp8,0,4.7705332438151045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,2,128,1,float16,float16,0,4.923856099446614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,2,128,1,float16,fp8,0,4.973194758097331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,4,128,1,float16,float16,0,5.302906672159831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,4,128,1,float16,fp8,0,5.59556261698405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,8,128,1,float16,float16,0,5.515157063802083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,8,128,1,float16,fp8,0,5.6942399342854815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,32,128,1,float16,float16,0,3.0998398462931314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,32,1,128,1,float16,float16,0,7.993978500366211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,1,128,1,float16,fp8,0,2.5568693478902182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,1,128,1,float16,float16,0,2.2476800282796225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,32,128,1,float16,fp8,0,3.356207847595215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,2,128,1,float16,float16,0,2.4012959798177085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,2,128,1,float16,fp8,0,2.5044426918029785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,4,128,1,float16,float16,0,2.621253331502279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,8,128,1,float16,float16,0,2.7224372227986655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,32,128,1,float16,float16,0,1.5805493990580242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,1,128,1,float16,float16,0,1.1394026279449463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,8,128,1,float16,fp8,0,2.9062986373901367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,32,128,1,float16,fp8,0,1.708122730255127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,32,4,128,1,float16,fp8,0,7.0257918039957685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,1,128,1,float16,fp8,0,1.2358026504516602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,4,128,1,float16,float16,0,1.3073866367340088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,2,128,1,float16,float16,0,1.2199947039286296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,2,128,1,float16,fp8,0,1.2822773456573486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,4,128,1,float16,fp8,0,1.4588853518168132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,8,128,1,float16,float16,0,1.3922719955444336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,32,128,1,float16,fp8,0,0.8561546802520752
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,32,128,1,float16,float16,0,0.7954826354980469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,32,8,128,1,float16,fp8,0,3.2073065439860025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,1,128,1,float16,float16,0,0.5865439971288046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,1,128,1,float16,fp8,0,0.6387466589609782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,2,128,1,float16,float16,0,0.6143146753311157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,32,128,1,float16,float16,0,0.4149386485417684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,8,128,1,float16,fp8,0,1.1536693572998047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,4,128,1,float16,float16,0,0.673525333404541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,2,128,1,float16,fp8,0,0.6594933271408081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,1,128,1,float16,fp8,0,0.33949867884318036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,4,128,1,float16,fp8,0,0.746229330698649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,32,128,1,float16,fp8,0,0.44278931617736816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,32,8,128,1,float16,float16,0,0.7095999717712402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,1,128,1,float16,float16,0,0.30340800682703656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,2,128,1,float16,float16,0,0.36955201625823975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,2,128,1,float16,fp8,0,0.3476853370666504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,4,128,1,float16,float16,0,0.3533066511154175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,1,128,1,float16,float16,0,0.17663466930389404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,4,128,1,float16,fp8,0,0.3924533526102702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,8,128,1,float16,float16,0,0.3728853464126587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,32,8,128,1,float16,fp8,0,0.3983146746953328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,32,128,1,float16,float16,0,0.23149333397547403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,32,128,1,float16,fp8,0,0.23918932676315308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,1,128,1,float16,fp8,0,0.1837386687596639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,2,128,1,float16,float16,0,0.17856534322102866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,2,128,1,float16,fp8,0,0.20329066117604574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,4,128,1,float16,float16,0,0.19131199518839517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,4,128,1,float16,fp8,0,0.21009600162506104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,8,128,1,float16,float16,0,0.20010133584340414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,32,8,128,1,float16,fp8,0,0.21590399742126465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,32,128,1,float16,float16,0,0.12691199779510498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,32,128,1,float16,fp8,0,0.13889599839846292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,1,128,1,float16,float16,0,0.0921493371327718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,1,128,1,float16,fp8,0,0.09902933239936829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,2,128,1,float16,float16,0,0.09892266988754272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,2,128,1,float16,fp8,0,0.10660800337791443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,4,128,1,float16,float16,0,0.10114133358001709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,4,128,1,float16,fp8,0,0.1113920013109843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,8,128,1,float16,float16,0,0.11463466286659241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,32,8,128,1,float16,fp8,0,0.12325867017110188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,1,128,1,float16,float16,0,2.7818241119384766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,2,128,1,float16,float16,0,2.909269332885742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,1,128,1,float16,fp8,0,5.052794774373372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,2,128,1,float16,fp8,0,3.1340001424153647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,4,128,1,float16,float16,0,3.2817598978678384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,4,128,1,float16,fp8,0,3.6177972157796225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,32,128,1,float16,float16,0,2.028992017110189
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,8,128,1,float16,float16,0,3.4094505310058594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,32,8,128,1,float16,fp8,0,3.745541254679362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,32,128,1,float16,fp8,0,2.3496373494466147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,1,128,1,float16,float16,0,2.332335948944092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,1,128,1,float16,fp8,0,1.5133387247721355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,2,128,1,float16,float16,0,1.4847839673360188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,2,128,1,float16,fp8,0,1.5735999743143718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,4,128,1,float16,float16,0,1.6670613288879395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,4,128,1,float16,fp8,0,1.8533706665039062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,8,128,1,float16,float16,0,1.7300853729248047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,32,8,128,1,float16,fp8,0,1.9030720392862956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,32,128,1,float16,float16,0,1.18503999710083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,32,128,1,float16,fp8,0,1.1049813429514568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,1,128,1,float16,float16,0,0.7085279623667399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,1,128,1,float16,fp8,0,0.7730240027109782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,2,128,1,float16,fp8,0,0.817039966583252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,2,128,1,float16,float16,0,0.7785173257191976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,4,128,1,float16,float16,0,0.8391626675923666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,4,128,1,float16,fp8,0,0.9391093254089355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,1,128,1,float16,float16,0,0.3723306655883789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,8,128,1,float16,float16,0,1.0680320262908936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,32,8,128,1,float16,fp8,0,0.9574346542358398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,32,128,1,float16,float16,0,0.522762656211853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,32,128,1,float16,fp8,0,0.5693279902140299
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,1,128,1,float16,fp8,0,0.4092053174972534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,2,128,1,float16,float16,0,0.39897600809733075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,8,128,1,float16,fp8,0,0.5024266640345255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,32,128,1,float16,fp8,0,0.3013919989267985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,32,128,1,float16,float16,0,0.27584532896677655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,2,128,1,float16,fp8,0,0.4196693499883016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,4,128,1,float16,float16,0,0.4325066804885864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,4,128,1,float16,fp8,0,0.527013341585795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,32,8,128,1,float16,float16,0,0.4519840081532796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,1,128,1,float16,float16,0,0.19826666514078775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,1,128,1,float16,fp8,0,0.2178186575571696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,2,128,1,float16,float16,0,0.21296000480651855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,2,128,1,float16,fp8,0,0.22564800580342612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,4,128,1,float16,float16,0,0.23202133178710938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,1,128,1,float16,float16,0,0.11110933621724446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,4,128,1,float16,fp8,0,0.2649493416150411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,8,128,1,float16,float16,0,0.24425599972407022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,32,8,128,1,float16,fp8,0,0.26925333340962726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,32,128,1,float16,float16,0,0.15371732910474142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,32,128,1,float16,fp8,0,0.16710933049519858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,1,128,1,float16,fp8,0,0.12172266840934753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,2,128,1,float16,float16,0,0.11896533767382304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,2,128,1,float16,fp8,0,0.12818666299184164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,4,128,1,float16,float16,0,0.1262079974015554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,4,128,1,float16,fp8,0,0.13901866475741068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,8,128,1,float16,fp8,0,0.1477173368136088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,32,8,128,1,float16,float16,0,0.1353653371334076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,32,128,1,float16,float16,0,0.08770133058230083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,32,128,1,float16,fp8,0,0.10004799564679463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,1,128,1,float16,float16,0,0.05964266757170359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,1,128,1,float16,fp8,0,0.06656533479690552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,2,128,1,float16,float16,0,0.06408533453941345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,2,128,1,float16,fp8,0,0.07091199855009715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,4,128,1,float16,float16,0,0.06871999800205231
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,4,128,1,float16,fp8,0,0.07814933359622955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,8,128,1,float16,fp8,0,0.08392000198364258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,32,8,128,1,float16,float16,0,0.0769760012626648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,1,128,1,float16,fp8,0,3.2153491973876953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,2,128,1,float16,float16,0,3.157562573750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,1,128,1,float16,float16,0,6.486703872680664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,2,128,1,float16,fp8,0,3.3442932764689126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,4,128,1,float16,float16,0,3.5374345779418945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,8,128,1,float16,float16,0,3.737600008646647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,32,128,1,float16,float16,0,2.2585652669270835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,8,128,1,float16,fp8,0,4.134341239929199
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,32,128,1,float16,fp8,0,2.5464107195536294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,1,128,1,float16,float16,0,2.077199935913086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,1,128,1,float16,fp8,0,1.6203360557556152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,32,4,128,1,float16,fp8,0,3.9960158665974936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,2,128,1,float16,float16,0,1.5961599349975586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,2,128,1,float16,fp8,0,1.691226641337077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,4,128,1,float16,float16,0,1.7803467114766438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,32,128,1,float16,float16,0,1.1443520387013753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,8,128,1,float16,float16,0,1.8972533543904622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,4,128,1,float16,fp8,0,2.0240373611450195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,1,128,1,float16,float16,0,0.7518453598022461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,1,128,1,float16,fp8,0,0.8266506989796957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,4,128,1,float16,float16,0,0.897589365641276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,32,128,1,float16,fp8,0,1.2797493139902751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,32,8,128,1,float16,fp8,0,2.945018768310547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,2,128,1,float16,float16,0,0.8288319905598959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,2,128,1,float16,fp8,0,0.8606506983439127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,4,128,1,float16,fp8,0,1.0252479712168376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,8,128,1,float16,float16,0,0.9757760365804037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,32,128,1,float16,float16,0,0.5814293225606283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,32,8,128,1,float16,fp8,0,1.076533317565918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,32,128,1,float16,fp8,0,0.6567786534627279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,1,128,1,float16,float16,0,0.40772799650828045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,1,128,1,float16,fp8,0,0.4304373264312744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,2,128,1,float16,float16,0,0.4195306698481242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,2,128,1,float16,fp8,0,0.4453706741333008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,4,128,1,float16,float16,0,0.47102399667104083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,4,128,1,float16,fp8,0,0.535530686378479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,8,128,1,float16,float16,0,0.5022240082422892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,32,8,128,1,float16,fp8,0,0.5502133369445801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,32,128,1,float16,float16,0,0.31351999441782635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,2,128,1,float16,fp8,0,0.2409706711769104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,32,128,1,float16,fp8,0,0.34357333183288574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,1,128,1,float16,float16,0,0.20635199546813965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,1,128,1,float16,fp8,0,0.22740799188613892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,2,128,1,float16,float16,0,0.22619734207789102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,4,128,1,float16,float16,0,0.251306672890981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,4,128,1,float16,fp8,0,0.27907200654347736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,8,128,1,float16,float16,0,0.2699093421300252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,32,8,128,1,float16,fp8,0,0.2955840031305949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,32,128,1,float16,float16,0,0.16587733229001364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,32,128,1,float16,fp8,0,0.1884373426437378
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,4,128,1,float16,float16,0,0.13743467132250467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,4,128,1,float16,fp8,0,0.15396799643834433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,1,128,1,float16,float16,0,0.10965333382288615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,1,128,1,float16,fp8,0,0.12787733475367227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,32,128,1,float16,fp8,0,0.10904533664385478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,2,128,1,float16,float16,0,0.1236853301525116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,2,128,1,float16,fp8,0,0.1334719955921173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,8,128,1,float16,float16,0,0.1443839967250824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,32,8,128,1,float16,fp8,0,0.16080000003178915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,32,128,1,float16,float16,0,0.09351999560991923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,1,128,1,float16,float16,0,0.062458669145902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,1,128,1,float16,fp8,0,0.07306133210659027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,2,128,1,float16,float16,0,0.07014933228492737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,2,128,1,float16,fp8,0,0.07453333338101704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,4,128,1,float16,float16,0,0.06919466455777486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,4,128,1,float16,fp8,0,0.08028799792130788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,1,128,1,float16,fp8,0,0.04177066683769226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,8,128,1,float16,float16,0,0.08231466511885326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,32,8,128,1,float16,fp8,0,0.09114133318265279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,32,128,1,float16,float16,0,0.05657066901524862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,32,128,1,float16,fp8,0,0.06414400041103363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,1,128,1,float16,float16,0,0.03668266783157984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,2,128,1,float16,float16,0,0.0401653324564298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,2,128,1,float16,fp8,0,0.04588800172011057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,4,128,1,float16,float16,0,0.04301333427429199
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,4,128,1,float16,fp8,0,0.04990933338801066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,8,128,1,float16,float16,0,0.04489600161711375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,32,8,128,1,float16,fp8,0,0.05136533578236898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,1,128,1,float16,float16,0,2.1700852711995444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,1,128,1,float16,fp8,0,2.4390026728312173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,2,128,1,float16,float16,0,2.4023359616597495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,2,128,1,float16,fp8,0,2.606325308481852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,4,128,1,float16,float16,0,2.794581413269043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,4,128,1,float16,fp8,0,4.121098518371582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,1,128,1,float16,float16,0,1.0858826637268066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,32,128,1,float16,float16,0,1.883855978647868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,1,128,1,float16,fp8,0,1.2301173210144043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,2,128,1,float16,float16,0,1.2163786888122559
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,4,128,1,float16,float16,0,1.3962079683939617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,8,128,1,float16,float16,0,3.0242932637532554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,4,128,1,float16,fp8,0,1.651093324025472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,32,128,1,float16,fp8,0,2.2519466082255044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,32,8,128,1,float16,fp8,0,3.337205251057943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,2,128,1,float16,fp8,0,1.413141409556071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,32,128,1,float16,float16,0,0.9525919755299886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,8,128,1,float16,float16,0,1.5104586283365886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,32,8,128,1,float16,fp8,0,1.683568000793457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,1,128,1,float16,float16,0,0.5552586714426676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,2,128,1,float16,fp8,0,0.697551965713501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,1,128,1,float16,fp8,0,0.6300319830576578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,4,128,1,float16,float16,0,0.7151146729787191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,32,128,1,float16,fp8,0,0.5833226839701334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,32,128,1,float16,float16,0,0.4915519952774048
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,8,128,1,float16,fp8,0,0.8908906777699789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,4,128,1,float16,fp8,0,0.8395199775695801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,8,128,1,float16,float16,0,0.7798026402791342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,32,128,1,float16,fp8,0,1.1388373374938965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,1,128,1,float16,fp8,0,0.3272479971249898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,1,128,1,float16,float16,0,0.2898079951604207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,2,128,1,float16,float16,0,0.3423253297805786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,8,128,1,float16,float16,0,0.39874664942423504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,4,128,1,float16,float16,0,0.37302935123443604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,2,128,1,float16,fp8,0,0.35011200110117596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,1,128,1,float16,float16,0,0.1551040013631185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,4,128,1,float16,fp8,0,0.4294826587041219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,32,2,128,1,float16,float16,0,0.6218346754709879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,32,128,1,float16,float16,0,0.2555253307024638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,32,8,128,1,float16,fp8,0,0.4574506680170695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,32,128,1,float16,fp8,0,0.31657065947850543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,1,128,1,float16,fp8,0,0.1760586698849996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,2,128,1,float16,float16,0,0.17101866006851196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,2,128,1,float16,fp8,0,0.18674665689468384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,32,128,1,float16,fp8,0,0.16505066553751627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,4,128,1,float16,float16,0,0.19666133324305216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,4,128,1,float16,fp8,0,0.22669865687688193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,1,128,1,float16,fp8,0,0.09922132889429729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,8,128,1,float16,float16,0,0.21304533878962198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,32,8,128,1,float16,fp8,0,0.24085867404937744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,32,128,1,float16,float16,0,0.137882669766744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,1,128,1,float16,float16,0,0.08711999654769897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,2,128,1,float16,float16,0,0.09514133135477702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,2,128,1,float16,fp8,0,0.10635200142860413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,4,128,1,float16,float16,0,0.10593600074450175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,4,128,1,float16,fp8,0,0.12351466218630473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,8,128,1,float16,float16,0,0.11527466773986816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,32,8,128,1,float16,fp8,0,0.13346667091051737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,32,128,1,float16,float16,0,0.07874133189519246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,32,128,1,float16,fp8,0,0.09483733773231506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,8,128,1,float16,float16,0,0.06634666522343953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,1,128,1,float16,float16,0,0.04897599915663401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,1,128,1,float16,fp8,0,0.05452266832192739
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,2,128,1,float16,float16,0,0.054192001620928444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,32,128,1,float16,fp8,0,0.05266133447488149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,2,128,1,float16,fp8,0,0.05719466507434845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,4,128,1,float16,float16,0,0.05489066739877065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,2,128,1,float16,float16,0,0.030778666337331135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,4,128,1,float16,fp8,0,0.06302933394908905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,32,8,128,1,float16,fp8,0,0.07578133543332417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,32,128,1,float16,float16,0,0.04714666803677877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,1,128,1,float16,float16,0,0.029696000119050343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,8,128,1,float16,fp8,0,0.0386559988061587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,1,128,1,float16,fp8,0,0.03461866577466329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,2,128,1,float16,fp8,0,0.03534399966398875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,1,128,1,float16,fp8,0,0.022463999688625336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,4,128,1,float16,float16,0,0.03310399999221166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,2,128,1,float16,float16,0,0.01972266659140587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,4,128,1,float16,fp8,0,0.03886933376391729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,32,8,128,1,float16,float16,0,0.033770665526390076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,32,128,1,float16,float16,0,0.023178666830062866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,32,128,1,float16,fp8,0,0.028202667832374573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,1,128,1,float16,float16,0,0.01958400011062622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,2,128,1,float16,fp8,0,0.023002666731675465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,4,128,1,float16,float16,0,0.021344001094500225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,4,128,1,float16,fp8,0,0.02478933334350586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,8,128,1,float16,float16,0,0.021551998953024547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,32,8,128,1,float16,fp8,0,0.025253333151340485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,1,128,1,float16,float16,0,0.9048799673716227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,1,128,1,float16,fp8,0,1.0350080331166585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,2,128,1,float16,float16,0,1.015935977300008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,2,128,1,float16,fp8,0,1.1225600242614746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,4,128,1,float16,float16,0,1.2350773016611736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,32,128,1,float16,float16,0,0.8739733695983887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,4,128,1,float16,fp8,0,1.456666628519694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,1,128,1,float16,float16,0,0.4628693262736003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,32,128,1,float16,fp8,0,0.9896533489227295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,8,128,1,float16,float16,0,1.343066692352295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,32,8,128,1,float16,fp8,0,1.5087040265401204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,1,128,1,float16,fp8,0,0.5285333395004272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,2,128,1,float16,float16,0,0.537557323773702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,2,128,1,float16,fp8,0,0.5771626631418864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,4,128,1,float16,float16,0,0.6237866481145223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,4,128,1,float16,fp8,0,0.7394186655680338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,1,128,1,float16,float16,0,0.24105066061019897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,32,128,1,float16,float16,0,0.44817066192626953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,8,128,1,float16,float16,0,0.6902080376942953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,32,8,128,1,float16,fp8,0,0.799402634302775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,32,128,1,float16,fp8,0,0.5124426682790121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,1,128,1,float16,fp8,0,0.2760480046272278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,2,128,1,float16,fp8,0,0.3002240061759949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,2,128,1,float16,float16,0,0.2758293350537618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,4,128,1,float16,float16,0,0.3261599938074748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,4,128,1,float16,fp8,0,0.3842560052871704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,32,128,1,float16,float16,0,0.23730667432149252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,8,128,1,float16,float16,0,0.3584746519724528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,32,128,1,float16,fp8,0,0.2656000057856242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,32,8,128,1,float16,fp8,0,0.41205867131551105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,1,128,1,float16,float16,0,0.13005333145459494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,1,128,1,float16,fp8,0,0.15044800440470377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,2,128,1,float16,float16,0,0.14892799655596414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,2,128,1,float16,fp8,0,0.16381866733233133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,4,128,1,float16,float16,0,0.17179199059804282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,4,128,1,float16,fp8,0,0.20232532421747842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,8,128,1,float16,float16,0,0.18710933128992716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,32,8,128,1,float16,fp8,0,0.21813867489496866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,32,128,1,float16,float16,0,0.12666133046150208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,32,128,1,float16,fp8,0,0.14614400267601013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,1,128,1,float16,float16,0,0.07523733377456665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,1,128,1,float16,fp8,0,0.08687999844551086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,8,128,1,float16,fp8,0,0.11954133709271748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,2,128,1,float16,float16,0,0.0820000022649765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,2,128,1,float16,fp8,0,0.09237866600354512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,4,128,1,float16,float16,0,0.09197333455085754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,1,128,1,float16,fp8,0,0.04650666813055674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,4,128,1,float16,fp8,0,0.11029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,32,8,128,1,float16,float16,0,0.10201066732406616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,32,128,1,float16,float16,0,0.07328533132870992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,32,128,1,float16,fp8,0,0.08451199531555176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,1,128,1,float16,float16,0,0.04106666644414266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,2,128,1,float16,float16,0,0.04514666895071665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,2,128,1,float16,fp8,0,0.051269332567850746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,4,128,1,float16,float16,0,0.046207999189694725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,4,128,1,float16,fp8,0,0.054058666030565895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,1,128,1,float16,fp8,0,0.028394666810830433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,8,128,1,float16,float16,0,0.057018667459487915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,2,128,1,float16,fp8,0,0.02974933385848999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,32,8,128,1,float16,fp8,0,0.06492266555627187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,32,128,1,float16,float16,0,0.041050667564074196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,32,128,1,float16,fp8,0,0.046122665206591286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,1,128,1,float16,float16,0,0.02421333392461141
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,2,128,1,float16,float16,0,0.025231999655564625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,4,128,1,float16,float16,0,0.027402666707833607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,1,128,1,float16,float16,0,0.01692266638080279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,1,128,1,float16,fp8,0,0.019167999426523846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,4,128,1,float16,fp8,0,0.03266133368015289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,8,128,1,float16,float16,0,0.028698667883872986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,32,8,128,1,float16,fp8,0,0.03307733436425527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,32,128,1,float16,float16,0,0.019813333948453266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,32,128,1,float16,fp8,0,0.02500266581773758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,2,128,1,float16,float16,0,0.016773333152135212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,2,128,1,float16,fp8,0,0.019632000476121902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,4,128,1,float16,float16,0,0.01851733277241389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,4,128,1,float16,fp8,0,0.02117866774400075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,8,128,1,float16,float16,0,0.018640000373125076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,2,128,1,float16,float16,0,0.01661866654952367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,32,8,128,1,float16,fp8,0,0.021407999098300934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,32,128,1,float16,float16,0,0.01770666614174843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,4,128,1,float16,fp8,0,0.01926400015751521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,32,128,1,float16,fp8,0,0.021856000026067097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,1,128,1,float16,float16,0,0.016234666109085083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,1,128,1,float16,fp8,0,0.019530666371186573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,2,128,1,float16,fp8,0,0.019071999937295914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,4,128,1,float16,float16,0,0.01722666621208191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,8,128,1,float16,float16,0,0.01646399994691213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,32,8,128,1,float16,fp8,0,0.019333332777023315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,1,128,1,float16,float16,0,0.4427786668141683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,1,128,1,float16,fp8,0,0.4774506489435832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,2,128,1,float16,float16,0,0.5018879969914755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,32,128,1,float16,fp8,0,0.45442668596903485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,2,128,1,float16,fp8,0,0.5226026773452759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,32,128,1,float16,float16,0,0.4296693404515584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,4,128,1,float16,float16,0,0.6204799811045328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,1,128,1,float16,float16,0,0.22947200139363608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,4,128,1,float16,fp8,0,0.6585013469060262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,8,128,1,float16,fp8,0,0.7557439804077148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,2,128,1,float16,float16,0,0.265338659286499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,32,8,128,1,float16,float16,0,0.6695253054300944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,2,128,1,float16,fp8,0,0.2758986751238505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,1,128,1,float16,fp8,0,0.2536960045496623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,4,128,1,float16,float16,0,0.32053865989049274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,4,128,1,float16,fp8,0,0.34304531415303546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,32,128,1,float16,fp8,0,0.2404586672782898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,8,128,1,float16,float16,0,0.33744001388549805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,32,128,1,float16,float16,0,0.22374399503072104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,32,8,128,1,float16,fp8,0,0.3946346839269002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,1,128,1,float16,float16,0,0.12446932991345723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,1,128,1,float16,fp8,0,0.13960533340771994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,2,128,1,float16,float16,0,0.14226133624712625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,2,128,1,float16,fp8,0,0.15152000387509665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,4,128,1,float16,float16,0,0.16671466827392578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,4,128,1,float16,fp8,0,0.18427199125289917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,8,128,1,float16,float16,0,0.17870400349299112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,1,128,1,float16,float16,0,0.07230400045712788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,32,8,128,1,float16,fp8,0,0.20922134319941202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,32,128,1,float16,float16,0,0.12354133526484172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,2,128,1,float16,fp8,0,0.0867733359336853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,32,128,1,float16,fp8,0,0.130213330189387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,1,128,1,float16,fp8,0,0.08084266881148021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,2,128,1,float16,float16,0,0.07890666524569194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,4,128,1,float16,float16,0,0.08654399712880452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,4,128,1,float16,fp8,0,0.10014399886131287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,8,128,1,float16,float16,0,0.10055466492970784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,32,8,128,1,float16,fp8,0,0.11354133486747742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,32,128,1,float16,float16,0,0.07019733389218648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,4,128,1,float16,float16,0,0.041696002086003624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,2,128,1,float16,float16,0,0.041749333341916404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,32,128,1,float16,fp8,0,0.07425066828727722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,1,128,1,float16,float16,0,0.036517334481080375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,1,128,1,float16,fp8,0,0.041050667564074196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,2,128,1,float16,fp8,0,0.04203199843565623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,4,128,1,float16,fp8,0,0.047279998660087585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,8,128,1,float16,float16,0,0.05610666672388712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,32,8,128,1,float16,fp8,0,0.06125866870085398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,32,128,1,float16,float16,0,0.039103999733924866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,32,128,1,float16,fp8,0,0.04137066751718521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,1,128,1,float16,float16,0,0.021231998999913532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,1,128,1,float16,fp8,0,0.02447466552257538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,2,128,1,float16,float16,0,0.022842665513356526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,2,128,1,float16,fp8,0,0.02593066543340683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,32,128,1,float16,float16,0,0.01860800012946129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,4,128,1,float16,float16,0,0.0249493345618248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,4,128,1,float16,fp8,0,0.028522667785485584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,8,128,1,float16,float16,0,0.025087999800841015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,2,128,1,float16,fp8,0,0.01801066721479098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,32,8,128,1,float16,fp8,0,0.029232000311215717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,32,128,1,float16,fp8,0,0.020949333906173706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,8,128,1,float16,float16,0,0.016901332885026932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,1,128,1,float16,float16,0,0.015599999576807022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,1,128,1,float16,fp8,0,0.017952000101407368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,2,128,1,float16,float16,0,0.01571200042963028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,4,128,1,float16,float16,0,0.016970666746298473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,4,128,1,float16,fp8,0,0.019466667125622433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,2,128,1,float16,float16,0,0.015077333897352219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,32,8,128,1,float16,fp8,0,0.019727999965349834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,32,128,1,float16,float16,0,0.01595199977358182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,32,128,1,float16,fp8,0,0.01830400029818217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,1,128,1,float16,float16,0,0.015178666760524115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,1,128,1,float16,fp8,0,0.017685333887736004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,2,128,1,float16,fp8,0,0.017562666287024815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,4,128,1,float16,float16,0,0.01488000030318896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,4,128,1,float16,fp8,0,0.01746133342385292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,8,128,1,float16,float16,0,0.015541333705186844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,32,8,128,1,float16,fp8,0,0.017743999759356182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,32,128,1,float16,float16,0,0.014981333166360855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,32,128,1,float16,fp8,0,0.017829333742459614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,1,128,1,float16,float16,0,0.015002666662136713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,1,128,1,float16,fp8,0,0.017445333302021027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,8,128,1,float16,fp8,0,0.01720000058412552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,2,128,1,float16,float16,0,0.014650666465361914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,2,128,1,float16,fp8,0,0.01743999992807706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,4,128,1,float16,float16,0,0.014869333555301031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,4,128,1,float16,fp8,0,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,32,8,128,1,float16,float16,0,0.014677333335081736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,1,128,1,float16,float16,0,0.23083732525507608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,1,128,1,float16,fp8,0,0.2526293396949768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,2,128,1,float16,float16,0,0.26818132400512695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,2,128,1,float16,fp8,0,0.2755519946416219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,4,128,1,float16,float16,0,0.3160426616668701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,32,128,1,float16,float16,0,0.24712532758712769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,4,128,1,float16,fp8,0,0.34279998143513996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,32,128,1,float16,fp8,0,0.23899734020233154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,1,128,1,float16,float16,0,0.12562132875124613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,8,128,1,float16,float16,0,0.3408106565475464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,2,128,1,float16,float16,0,0.14134933551152548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,1,128,1,float16,fp8,0,0.14149333039919534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,2,128,1,float16,fp8,0,0.15265599886576334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,32,8,128,1,float16,fp8,0,0.386624018351237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,4,128,1,float16,fp8,0,0.18141865730285645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,4,128,1,float16,float16,0,0.1650773286819458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,8,128,1,float16,float16,0,0.18042133251825967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,32,8,128,1,float16,fp8,0,0.20683733622233072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,32,128,1,float16,float16,0,0.13177067041397095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,32,128,1,float16,fp8,0,0.12682132919629416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,1,128,1,float16,float16,0,0.07217066486676534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,1,128,1,float16,fp8,0,0.08117866516113281
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,2,128,1,float16,float16,0,0.08011200030644734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,2,128,1,float16,fp8,0,0.08741333087285359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,4,128,1,float16,float16,0,0.08627200126647949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,4,128,1,float16,fp8,0,0.1002453366915385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,8,128,1,float16,float16,0,0.09896533687909444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,32,8,128,1,float16,fp8,0,0.11339199542999268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,32,128,1,float16,float16,0,0.07438399891058604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,32,128,1,float16,fp8,0,0.07165333131949107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,1,128,1,float16,float16,0,0.036288000643253326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,1,128,1,float16,fp8,0,0.040287998815377556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,2,128,1,float16,float16,0,0.04177600145339966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,2,128,1,float16,fp8,0,0.04489600161711375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,4,128,1,float16,float16,0,0.041034666200478874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,4,128,1,float16,fp8,0,0.04771199822425842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,8,128,1,float16,float16,0,0.05486933390299479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,32,8,128,1,float16,fp8,0,0.05994133154551188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,32,128,1,float16,float16,0,0.04241600135962168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,32,128,1,float16,fp8,0,0.037418665985266365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,1,128,1,float16,float16,0,0.021850667893886566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,1,128,1,float16,fp8,0,0.024469333390394848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,8,128,1,float16,float16,0,0.025797332326571148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,2,128,1,float16,float16,0,0.022448000808556873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,2,128,1,float16,fp8,0,0.025941332181294758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,4,128,1,float16,float16,0,0.024677333732446034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,4,128,1,float16,fp8,0,0.028991999725500744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,32,8,128,1,float16,fp8,0,0.029487999776999157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,32,128,1,float16,float16,0,0.01982933282852173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,32,128,1,float16,fp8,0,0.020213333268960316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,1,128,1,float16,float16,0,0.01563199982047081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,1,128,1,float16,fp8,0,0.0180479995906353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,2,128,1,float16,fp8,0,0.017935999979575474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,2,128,1,float16,float16,0,0.015552000453074774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,4,128,1,float16,float16,0,0.017338667064905167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,4,128,1,float16,fp8,0,0.01934933289885521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,1,128,1,float16,fp8,0,0.017407999684413273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,8,128,1,float16,float16,0,0.01691199963291486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,32,8,128,1,float16,fp8,0,0.019632000476121902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,32,128,1,float16,float16,0,0.014111999422311783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,32,128,1,float16,fp8,0,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,1,128,1,float16,float16,0,0.014848000059525171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,2,128,1,float16,float16,0,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,2,128,1,float16,fp8,0,0.017642666896184284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,4,128,1,float16,float16,0,0.01492799942692121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,4,128,1,float16,fp8,0,0.017130666722853977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,8,128,1,float16,float16,0,0.015749332805474598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,32,8,128,1,float16,fp8,0,0.017871999492247898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,32,128,1,float16,float16,0,0.01332266628742218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,32,128,1,float16,fp8,0,0.013781332721312841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,1,128,1,float16,float16,0,0.014970666418472925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,1,128,1,float16,fp8,0,0.017024000485738117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,2,128,1,float16,float16,0,0.014394666999578476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,2,128,1,float16,fp8,0,0.017221332838137943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,4,128,1,float16,float16,0,0.014655999839305878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,1,128,1,float16,float16,0,0.013786666095256805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,4,128,1,float16,fp8,0,0.01661866654952367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,8,128,1,float16,float16,0,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,32,8,128,1,float16,fp8,0,0.017450666675964992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,32,128,1,float16,float16,0,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,32,128,1,float16,fp8,0,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,8,128,1,float16,float16,0,0.013973332941532135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,1,128,1,float16,fp8,0,0.016645333419243496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,2,128,1,float16,float16,0,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,2,128,1,float16,fp8,0,0.017008000363906223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,4,128,1,float16,float16,0,0.013845333208640417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,4,128,1,float16,fp8,0,0.015674666812022526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,1,128,1,float16,float16,0,0.12417067090670268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,32,8,128,1,float16,fp8,0,0.016117333124081295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,1,128,1,float16,fp8,0,0.14004266262054443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,2,128,1,float16,float16,0,0.14089066783587137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,2,128,1,float16,fp8,0,0.15308800339698792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,4,128,1,float16,float16,0,0.16275733709335327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,4,128,1,float16,fp8,0,0.18104000886281332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,8,128,1,float16,float16,0,0.17811733484268188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,32,8,128,1,float16,fp8,0,0.20713067054748535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,32,128,1,float16,float16,0,0.18121065696080527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,32,128,1,float16,fp8,0,0.17374932765960693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,1,128,1,float16,float16,0,0.0707946668068568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,1,128,1,float16,fp8,0,0.08088000118732452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,2,128,1,float16,float16,0,0.07916266719500224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,2,128,1,float16,fp8,0,0.08772800366083781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,4,128,1,float16,float16,0,0.08514666557312012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,4,128,1,float16,fp8,0,0.09935999910036723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,8,128,1,float16,float16,0,0.09923199812571208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,32,128,1,float16,float16,0,0.09666132926940918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,32,8,128,1,float16,fp8,0,0.11288533608118693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,32,128,1,float16,fp8,0,0.09526933232943217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,1,128,1,float16,float16,0,0.03585066646337509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,1,128,1,float16,fp8,0,0.03935466706752777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,2,128,1,float16,float16,0,0.04072533299525579
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,2,128,1,float16,fp8,0,0.04357333481311798
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,32,128,1,float16,fp8,0,0.04827733337879181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,4,128,1,float16,float16,0,0.04154133299986521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,4,128,1,float16,fp8,0,0.04808000226815542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,8,128,1,float16,float16,0,0.05648000041643778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,2,128,1,float16,fp8,0,0.02595199892918269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,32,8,128,1,float16,fp8,0,0.061424002051353455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,32,128,1,float16,float16,0,0.054058666030565895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,4,128,1,float16,fp8,0,0.028650666276613872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,1,128,1,float16,float16,0,0.02088533341884613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,1,128,1,float16,fp8,0,0.02418133368094762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,2,128,1,float16,float16,0,0.022111999491850536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,4,128,1,float16,float16,0,0.024725332856178284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,8,128,1,float16,float16,0,0.025637333591779072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,32,8,128,1,float16,fp8,0,0.029205332199732464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,32,128,1,float16,float16,0,0.02585600068171819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,32,128,1,float16,fp8,0,0.026554666459560394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,4,128,1,float16,fp8,0,0.019253333409627277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,1,128,1,float16,float16,0,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,1,128,1,float16,fp8,0,0.017898666361967724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,8,128,1,float16,fp8,0,0.019551999866962433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,2,128,1,float16,float16,0,0.015557333827018738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,2,128,1,float16,fp8,0,0.01794133335351944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,4,128,1,float16,float16,0,0.0173333336909612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,32,8,128,1,float16,float16,0,0.016805333395799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,32,128,1,float16,float16,0,0.017242666333913803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,32,128,1,float16,fp8,0,0.017551999539136887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,4,128,1,float16,float16,0,0.015317333241303762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,1,128,1,float16,float16,0,0.014805333067973455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,1,128,1,float16,fp8,0,0.01748266691962878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,2,128,1,float16,float16,0,0.014954666296641031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,2,128,1,float16,fp8,0,0.01739199956258138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,4,128,1,float16,fp8,0,0.01725333308180173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,1,128,1,float16,float16,0,0.014463999619086584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,8,128,1,float16,float16,0,0.015050667027632395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,2,128,1,float16,float16,0,0.014671999961137772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,32,8,128,1,float16,fp8,0,0.01740266631046931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,32,128,1,float16,float16,0,0.013189333180586496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,32,128,1,float16,fp8,0,0.013733333597580591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,1,128,1,float16,fp8,0,0.01718933383623759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,2,128,1,float16,fp8,0,0.016938666502634685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,4,128,1,float16,float16,0,0.014490666488806406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,4,128,1,float16,fp8,0,0.01701333373785019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,8,128,1,float16,float16,0,0.014522666732470194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,32,8,128,1,float16,fp8,0,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,2,128,1,float16,fp8,0,0.0169813334941864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,4,128,1,float16,float16,0,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,32,128,1,float16,float16,0,0.012709333250919977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,8,128,1,float16,float16,0,0.014314666390419006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,32,128,1,float16,fp8,0,0.013264000415802002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,1,128,1,float16,float16,0,0.013637332866589228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,2,128,1,float16,float16,0,0.013797332843144735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,1,128,1,float16,fp8,0,0.01587733378012975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,4,128,1,float16,fp8,0,0.015487999965747198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,32,8,128,1,float16,fp8,0,0.016837333639462788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,32,128,1,float16,float16,0,0.01249066616098086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,32,128,1,float16,fp8,0,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,1,128,1,float16,float16,0,0.013770667215188345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,8,128,1,float16,float16,0,0.013653332988421122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,1,128,1,float16,fp8,0,0.015482666591803232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,2,128,1,float16,float16,0,0.013573333621025085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,2,128,1,float16,fp8,0,0.015829333414634068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,4,128,1,float16,float16,0,0.013701333353916803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,4,128,1,float16,fp8,0,0.015546667079130808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,32,8,128,1,float16,fp8,0,0.01581866666674614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,1,128,1,float16,float16,0,0.07075199981530507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,1,128,1,float16,fp8,0,0.08107199768225352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,2,128,1,float16,float16,0,0.0792799989382426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,2,128,1,float16,fp8,0,0.08721066514650981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,8,128,1,float16,float16,0,0.10849066575368245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,4,128,1,float16,float16,0,0.085125337044398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,4,128,1,float16,fp8,0,0.09864532947540283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,32,128,1,float16,float16,0,0.14805866281191507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,32,8,128,1,float16,fp8,0,0.10822400450706482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,32,128,1,float16,fp8,0,0.14321600397427878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,1,128,1,float16,float16,0,0.03645866612593333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,1,128,1,float16,fp8,0,0.03999999910593033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,2,128,1,float16,float16,0,0.04114133367935816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,2,128,1,float16,fp8,0,0.04479999840259552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,4,128,1,float16,float16,0,0.04164266586303711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,4,128,1,float16,fp8,0,0.04766400158405304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,8,128,1,float16,float16,0,0.06182933350404104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,32,128,1,float16,float16,0,0.07909333209196727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,32,8,128,1,float16,fp8,0,0.05490666627883911
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,32,128,1,float16,fp8,0,0.07356800138950348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,1,128,1,float16,float16,0,0.020975999534130096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,1,128,1,float16,fp8,0,0.024149333437283833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,2,128,1,float16,float16,0,0.022863999009132385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,32,128,1,float16,float16,0,0.038634667793909706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,8,128,1,float16,fp8,0,0.0284853329261144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,2,128,1,float16,fp8,0,0.025914666553338368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,4,128,1,float16,float16,0,0.024373332659403484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,4,128,1,float16,fp8,0,0.028421332438786823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,2,128,1,float16,fp8,0,0.018207999567190807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,32,8,128,1,float16,float16,0,0.02769600103298823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,32,128,1,float16,fp8,0,0.03827200084924698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,1,128,1,float16,float16,0,0.015626666446526844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,1,128,1,float16,fp8,0,0.017887999614079792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,2,128,1,float16,float16,0,0.01590399940808614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,4,128,1,float16,float16,0,0.01666133354107539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,4,128,1,float16,fp8,0,0.01921066641807556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,1,128,1,float16,fp8,0,0.017535999417304993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,8,128,1,float16,float16,0,0.018325333793958027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,32,8,128,1,float16,fp8,0,0.018858666221300762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,32,128,1,float16,float16,0,0.023733332753181458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,32,128,1,float16,fp8,0,0.024127999941507976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,1,128,1,float16,float16,0,0.014538666854302088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,2,128,1,float16,float16,0,0.015098666151364645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,2,128,1,float16,fp8,0,0.01708799973130226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,4,128,1,float16,float16,0,0.01533866673707962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,1,128,1,float16,float16,0,0.014368000129858652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,1,128,1,float16,fp8,0,0.017194667210181553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,4,128,1,float16,fp8,0,0.017535999417304993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,8,128,1,float16,float16,0,0.013381333400805792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,32,8,128,1,float16,fp8,0,0.013728000223636627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,32,128,1,float16,float16,0,0.016677333662907284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,32,128,1,float16,fp8,0,0.017008000363906223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,8,128,1,float16,fp8,0,0.01310933381319046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,2,128,1,float16,float16,0,0.015034666905800501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,2,128,1,float16,fp8,0,0.01720533271630605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,32,128,1,float16,fp8,0,0.013354666531085968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,4,128,1,float16,float16,0,0.014282666146755219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,4,128,1,float16,fp8,0,0.017338667064905167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,32,8,128,1,float16,float16,0,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,32,128,1,float16,float16,0,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,1,128,1,float16,float16,0,0.013951999445756277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,1,128,1,float16,fp8,0,0.01590399940808614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,2,128,1,float16,float16,0,0.014730667074521383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,2,128,1,float16,fp8,0,0.01598400001724561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,4,128,1,float16,float16,0,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,4,128,1,float16,fp8,0,0.01552533358335495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,8,128,1,float16,float16,0,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,32,8,128,1,float16,fp8,0,0.013125333935022354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,32,128,1,float16,float16,0,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,32,128,1,float16,fp8,0,0.01321600005030632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,1,128,1,float16,float16,0,0.013610667238632837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,1,128,1,float16,fp8,0,0.015802666544914246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,2,128,1,float16,float16,0,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,2,128,1,float16,fp8,0,0.01565333331624667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,4,128,1,float16,float16,0,0.013546666751305262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,4,128,1,float16,fp8,0,0.015530666957298914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,8,128,1,float16,float16,0,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,32,8,128,1,float16,fp8,0,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,32,128,1,float16,fp8,0,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,32,128,1,float16,float16,0,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,1,128,1,float16,float16,0,0.013376000026861826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,1,128,1,float16,fp8,0,0.015381333728631338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,2,128,1,float16,float16,0,0.013461332768201828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,2,128,1,float16,fp8,0,0.015439999600251516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,4,128,1,float16,float16,0,0.013301332791646322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,4,128,1,float16,fp8,0,0.015109332899252573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,8,128,1,float16,float16,0,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,0,0.023200000325838726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,32,8,128,1,float16,fp8,0,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,0,0.021226666867733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,0,0.027642667293548584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,0,0.04035733391841253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,0,0.029717333614826202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,0,0.04264533519744873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,0,0.07108800113201141
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,0,0.06643733382225037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,32,128,1,float16,float16,0,0.11309333642323811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,32,128,1,float16,fp8,0,0.11955733100573222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,0,0.014271999398867289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,0,0.015397333850463232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,0,0.017488000293572743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,0,0.01903466631968816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,0,0.024031999210516613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,0,0.026015999416510265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,0,0.013237333546082178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,0,0.03684266656637192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,0,0.040175999204317726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,32,128,1,float16,float16,0,0.062090665102005005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,32,128,1,float16,fp8,0,0.06607466439406078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,0,0.012495999534924826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,0,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,0,0.02497066557407379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,32,128,1,float16,float16,0,0.03621866554021835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,0,0.014245333770910898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,0,0.016773333152135212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,0,0.01775466650724411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,0,0.023397333920001984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,0,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,32,128,1,float16,fp8,0,0.03869866579771042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,0,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,0,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,0,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,32,128,1,float16,fp8,0,0.025072000920772552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,0,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,0,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,0,0.01179733375708262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,0,0.016250666230916977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,0,0.017407999684413273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,0,0.012741333494583765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,32,128,1,float16,float16,0,0.023408000667889912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,0,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,32,128,1,float16,float16,0,0.0162773331006368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,0,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,0,0.012784000486135483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,0,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,0,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,0,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,32,128,1,float16,fp8,0,0.017301333447297413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,0,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,0,0.01098666712641716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,0,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,0,0.01181866725285848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,0,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,0,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,0,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,0,0.012906666845083237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,32,128,1,float16,float16,0,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,32,128,1,float16,fp8,0,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,0,0.011663999408483505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,0,0.010858666151762009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,0,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,0,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,0,0.011786667009194693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,0,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,0,0.011642667154471079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,0,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,32,128,1,float16,float16,0,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,32,128,1,float16,fp8,0,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,0,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,0,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,0,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,0,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,0,0.011221333096424738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,0,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,0,0.011301333705584208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,0,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,32,128,1,float16,float16,0,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,32,128,1,float16,fp8,0,0.013050666699806849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,0,0.010821333775917688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,0,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,0,0.010944000134865442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,0,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,0,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,0,0.012229333321253458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,0,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,0,0.012256000190973282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,1,128,1,float16,fp8,0,20.498911539713543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,1,128,1,float16,float16,0,22.993370056152344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,2,128,1,float16,fp8,0,20.671109517415363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,2,128,1,float16,float16,0,23.165812174479168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,4,128,1,float16,fp8,0,21.56164805094401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,4,128,1,float16,float16,0,24.11400604248047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,8,128,1,float16,float16,0,23.3025385538737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,24,128,1,float16,float16,0,12.3243408203125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,24,128,1,float16,fp8,0,11.116378784179688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,1,128,1,float16,fp8,0,10.326896031697592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,1,128,1,float16,float16,0,11.98483149210612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,2,128,1,float16,float16,0,11.952810923258463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,2,128,1,float16,fp8,0,10.413007736206055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,4,128,1,float16,float16,0,12.430272420247396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,24,8,128,1,float16,fp8,0,40.25060272216797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,24,128,1,float16,float16,0,5.963706970214844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,1,128,1,float16,float16,0,5.657381057739258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,24,128,1,float16,fp8,0,5.958031972249349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,4,128,1,float16,fp8,0,10.802555084228516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,8,128,1,float16,float16,0,12.410666147867838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,1,128,1,float16,fp8,0,5.429178873697917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,24,8,128,1,float16,fp8,0,10.80990982055664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,2,128,1,float16,fp8,0,5.673557281494141
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,24,128,1,float16,float16,0,2.8963521321614585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,4,128,1,float16,fp8,0,5.463669459025065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,4,128,1,float16,float16,0,5.8215891520182295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,24,128,1,float16,fp8,0,2.8764959971110025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,8,128,1,float16,fp8,0,5.9284108479817705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,8,128,1,float16,float16,0,5.791274388631185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,1,128,1,float16,float16,0,2.686800003051758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,1,128,1,float16,fp8,0,2.885413487752279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,2,128,1,float16,float16,0,2.720911979675293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,2,128,1,float16,fp8,0,3.018085479736328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,4,128,1,float16,float16,0,2.8426345189412436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,24,2,128,1,float16,float16,0,15.789466857910156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,4,128,1,float16,fp8,0,2.796527862548828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,8,128,1,float16,float16,0,2.8020105361938477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,24,8,128,1,float16,fp8,0,2.9299891789754233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,1,128,1,float16,float16,0,13.49172846476237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,1,128,1,float16,fp8,0,11.880507151285807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,2,128,1,float16,fp8,0,11.935674031575521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,2,128,1,float16,float16,0,13.708688100179037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,4,128,1,float16,fp8,0,12.500373840332031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,8,128,1,float16,float16,0,13.65280024210612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,8,128,1,float16,fp8,0,12.577754974365234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,24,128,1,float16,float16,0,6.877344131469727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,1,128,1,float16,float16,0,6.356880187988281
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,24,128,1,float16,fp8,0,6.530997594197591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,1,128,1,float16,fp8,0,6.146432240804036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,2,128,1,float16,float16,0,7.076053619384766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,2,128,1,float16,fp8,0,6.001818974812825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,4,128,1,float16,float16,0,7.139018376668294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,24,128,1,float16,float16,0,3.408602714538574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,4,128,1,float16,fp8,0,6.56284777323405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,1,128,1,float16,float16,0,2.993957201639811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,8,128,1,float16,float16,0,6.799274444580078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,24,128,1,float16,fp8,0,3.4483200709025064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,1,128,1,float16,fp8,0,3.6702826817830405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,24,8,128,1,float16,fp8,0,6.320341110229492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,2,128,1,float16,float16,0,3.4684425989786782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,2,128,1,float16,fp8,0,3.0504318873087564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,4,128,1,float16,float16,0,3.2174720764160156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,4,128,1,float16,fp8,0,3.356522560119629
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,24,128,1,float16,float16,0,1.7147572835286458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,8,128,1,float16,float16,0,3.276159922281901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,24,128,1,float16,fp8,0,1.7773226102193196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,1,128,1,float16,float16,0,1.5535306930541992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,24,8,128,1,float16,fp8,0,3.2201013565063477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,1,128,1,float16,fp8,0,1.666810671488444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,2,128,1,float16,float16,0,1.5984692573547363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,2,128,1,float16,fp8,0,1.5829866727193196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,4,128,1,float16,float16,0,1.6509973208109539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,4,128,1,float16,fp8,0,1.6580212910970051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,8,128,1,float16,float16,0,1.6533066431681316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,24,8,128,1,float16,fp8,0,1.663962682088216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,24,4,128,1,float16,float16,0,47.855509440104164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,1,128,1,float16,float16,0,9.30615488688151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,1,128,1,float16,fp8,0,8.326522827148438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,2,128,1,float16,float16,0,10.098362604777018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,2,128,1,float16,fp8,0,8.530314763387045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,4,128,1,float16,float16,0,9.788437525431315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,4,128,1,float16,fp8,0,9.056975682576498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,8,128,1,float16,float16,0,9.657370885213217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,24,128,1,float16,float16,0,4.973552068074544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,1,128,1,float16,float16,0,4.392357190450032
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,24,128,1,float16,fp8,0,4.953189214070638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,2,128,1,float16,float16,0,4.5058027903238935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,2,128,1,float16,fp8,0,4.254096031188965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,4,128,1,float16,float16,0,4.69105593363444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,24,8,128,1,float16,fp8,0,8.961407979329428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,1,128,1,float16,fp8,0,8.279087702433268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,24,128,1,float16,float16,0,2.393338680267334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,4,128,1,float16,fp8,0,4.69596799214681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,24,128,1,float16,fp8,0,2.396752039591471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,1,128,1,float16,float16,0,2.1542720794677734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,8,128,1,float16,float16,0,4.80130132039388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,24,8,128,1,float16,fp8,0,4.526522636413574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,1,128,1,float16,fp8,0,2.130671977996826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,2,128,1,float16,fp8,0,2.1883412996927896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,4,128,1,float16,float16,0,2.288858731587728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,2,128,1,float16,float16,0,6.325381596883138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,4,128,1,float16,fp8,0,2.27620792388916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,24,128,1,float16,float16,0,1.2342933019002278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,8,128,1,float16,float16,0,2.2995200157165527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,24,128,1,float16,fp8,0,1.242037296295166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,1,128,1,float16,float16,0,1.1038080056508381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,24,8,128,1,float16,fp8,0,2.431466738382975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,1,128,1,float16,fp8,0,1.2588693300882976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,2,128,1,float16,fp8,0,1.1179626782735188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,2,128,1,float16,float16,0,1.7268479665120442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,4,128,1,float16,float16,0,1.1713013648986816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,4,128,1,float16,fp8,0,1.2425226370493572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,8,128,1,float16,float16,0,1.1695786317189534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,24,8,128,1,float16,fp8,0,1.2324053446451824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,1,128,1,float16,fp8,0,10.905792236328125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,1,128,1,float16,float16,0,12.223504384358725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,2,128,1,float16,float16,0,12.608688354492188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,4,128,1,float16,fp8,0,11.846453348795572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,4,128,1,float16,float16,0,13.372154235839844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,8,128,1,float16,float16,0,12.896191914876303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,8,128,1,float16,fp8,0,11.801695505777994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,24,128,1,float16,float16,0,6.320565541585286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,1,128,1,float16,float16,0,5.878053029378255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,24,2,128,1,float16,fp8,0,18.902037302652996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,24,128,1,float16,fp8,0,6.266202926635742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,1,128,1,float16,fp8,0,5.449136098225911
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,2,128,1,float16,float16,0,5.797333399454753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,2,128,1,float16,fp8,0,5.575802485148112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,4,128,1,float16,float16,0,6.059706370035808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,24,128,1,float16,float16,0,3.1466827392578125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,24,128,1,float16,fp8,0,3.1456480026245117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,1,128,1,float16,float16,0,2.8377014795939126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,4,128,1,float16,fp8,0,6.2126509348551435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,1,128,1,float16,fp8,0,2.7695465087890625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,8,128,1,float16,float16,0,6.4716746012369795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,2,128,1,float16,float16,0,2.929562568664551
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,2,128,1,float16,fp8,0,3.0487680435180664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,4,128,1,float16,float16,0,3.0225280125935874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,4,128,1,float16,fp8,0,3.1384960810343423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,8,128,1,float16,float16,0,3.018650690714518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,24,128,1,float16,float16,0,1.7072319984436035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,24,8,128,1,float16,fp8,0,10.282725016276041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,1,128,1,float16,float16,0,1.4070933659871419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,24,8,128,1,float16,fp8,0,3.047727902730306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,2,128,1,float16,float16,0,1.4411199887593586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,24,128,1,float16,fp8,0,1.8208959897359211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,1,128,1,float16,fp8,0,1.525386651357015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,2,128,1,float16,fp8,0,1.5839625994364421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,4,128,1,float16,float16,0,1.528506596883138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,8,128,1,float16,float16,0,1.533247947692871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,1,128,1,float16,float16,0,0.7299946943918864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,24,128,1,float16,float16,0,0.8374666372934977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,4,128,1,float16,fp8,0,2.2236533164978027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,24,8,128,1,float16,fp8,0,1.5653120676676433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,2,128,1,float16,fp8,0,0.7751573721567789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,4,128,1,float16,fp8,0,0.8142346541086832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,24,128,1,float16,fp8,0,0.884938637415568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,1,128,1,float16,fp8,0,0.7750506401062012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,2,128,1,float16,float16,0,0.7491199970245361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,4,128,1,float16,float16,0,0.7870933214823405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,8,128,1,float16,float16,0,0.8976799647013346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,24,8,128,1,float16,fp8,0,0.8223413626352946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,1,128,1,float16,fp8,0,6.401418685913086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,1,128,1,float16,float16,0,7.121327718098958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,2,128,1,float16,float16,0,6.760778427124023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,2,128,1,float16,fp8,0,6.494784037272136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,4,128,1,float16,float16,0,7.342976252237956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,4,128,1,float16,fp8,0,7.224474589029948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,8,128,1,float16,fp8,0,7.06110954284668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,1,128,1,float16,float16,0,3.25324281056722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,24,128,1,float16,float16,0,3.777594566345215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,1,128,1,float16,fp8,0,3.1941760381062827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,24,128,1,float16,fp8,0,3.823573430379232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,2,128,1,float16,float16,0,3.368197441101074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,2,128,1,float16,fp8,0,3.26093324025472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,4,128,1,float16,float16,0,3.4823147455851235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,24,128,1,float16,float16,0,1.8824532826741536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,4,128,1,float16,fp8,0,3.7055253982543945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,8,128,1,float16,float16,0,3.568533261617025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,24,128,1,float16,fp8,0,2.0167412757873535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,24,8,128,1,float16,fp8,0,3.5650933583577475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,1,128,1,float16,float16,0,1.6042613983154297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,2,128,1,float16,float16,0,1.651290734608968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,1,128,1,float16,fp8,0,1.7095252672831218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,2,128,1,float16,fp8,0,1.7610507011413574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,4,128,1,float16,float16,0,1.7743733723958333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,4,128,1,float16,fp8,0,1.8089067141215007
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,8,128,1,float16,float16,0,1.7926506996154785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,24,128,1,float16,float16,0,0.971839984258016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,24,8,128,1,float16,fp8,0,1.8546560605367024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,24,128,1,float16,fp8,0,1.015781323115031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,1,128,1,float16,float16,0,0.8263999621073405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,1,128,1,float16,fp8,0,0.8421440124511719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,2,128,1,float16,float16,0,0.8531306584676107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,2,128,1,float16,fp8,0,0.8810133139292399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,4,128,1,float16,float16,0,0.9046719868977865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,4,128,1,float16,fp8,0,0.9595093727111816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,24,8,128,1,float16,float16,0,26.79474131266276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,8,128,1,float16,float16,0,0.9247519969940186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,24,8,128,1,float16,fp8,0,0.9495680332183838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,24,128,1,float16,float16,0,0.5130666494369507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,24,128,1,float16,fp8,0,0.5454506476720175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,1,128,1,float16,float16,0,0.43485331535339355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,1,128,1,float16,fp8,0,0.45558400948842365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,2,128,1,float16,float16,0,0.44682665665944415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,4,128,1,float16,float16,0,0.4777119954427083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,2,128,1,float16,fp8,0,0.6899466514587402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,4,128,1,float16,fp8,0,0.49931732813517254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,8,128,1,float16,float16,0,0.48950934410095215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,24,8,128,1,float16,fp8,0,0.5217546621958414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,1,128,1,float16,float16,0,5.910943984985352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,1,128,1,float16,fp8,0,6.03605842590332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,2,128,1,float16,float16,0,6.41598383585612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,2,128,1,float16,fp8,0,6.28875732421875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,4,128,1,float16,fp8,0,6.9972585042317705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,8,128,1,float16,float16,0,7.061391830444336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,24,128,1,float16,float16,0,3.688394546508789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,8,128,1,float16,fp8,0,7.16431999206543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,1,128,1,float16,float16,0,2.960543950398763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,24,128,1,float16,fp8,0,3.8066345850626626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,1,128,1,float16,fp8,0,3.035696029663086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,2,128,1,float16,float16,0,3.1254027684529624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,2,128,1,float16,fp8,0,3.3130772908528647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,4,128,1,float16,float16,0,3.444133440653483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,4,128,1,float16,fp8,0,3.5289974212646484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,24,128,1,float16,float16,0,1.8579893112182617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,24,128,1,float16,fp8,0,1.9526774088541667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,1,128,1,float16,float16,0,1.4944000244140625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,8,128,1,float16,float16,0,3.4510933558146157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,24,8,128,1,float16,fp8,0,3.5141706466674805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,2,128,1,float16,float16,0,1.5631146430969238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,2,128,1,float16,fp8,0,1.6498986879984539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,1,128,1,float16,fp8,0,1.597200075785319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,4,128,1,float16,float16,0,1.7316746711730957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,4,128,1,float16,fp8,0,1.7930560111999512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,24,128,1,float16,float16,0,0.9488639831542969
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,8,128,1,float16,float16,0,1.7276479403177898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,24,128,1,float16,fp8,0,1.0087680021921794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,24,8,128,1,float16,fp8,0,1.811903953552246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,1,128,1,float16,float16,0,0.7656906445821127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,1,128,1,float16,fp8,0,0.8046133518218994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,2,128,1,float16,float16,0,0.8012853463490804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,24,4,128,1,float16,float16,0,23.317487080891926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,2,128,1,float16,fp8,0,0.8279946645100912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,4,128,1,float16,float16,0,0.8712800343831381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,4,128,1,float16,fp8,0,0.9396373430887858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,1,128,1,float16,float16,0,0.4678719838460286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,8,128,1,float16,float16,0,0.8917706807454427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,24,8,128,1,float16,fp8,0,0.9577546914418539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,24,128,1,float16,float16,0,0.4983839988708496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,24,128,1,float16,fp8,0,0.5335413217544556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,1,128,1,float16,fp8,0,0.42609067757924396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,2,128,1,float16,float16,0,0.41602134704589844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,2,128,1,float16,fp8,0,0.4338719844818115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,4,128,1,float16,float16,0,0.5843093395233154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,4,128,1,float16,fp8,0,0.49212801456451416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,8,128,1,float16,float16,0,0.4673973321914673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,24,8,128,1,float16,fp8,0,0.5047093232472738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,24,128,1,float16,float16,0,0.2739199995994568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,24,128,1,float16,fp8,0,0.2911520004272461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,1,128,1,float16,float16,0,0.22122132778167725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,4,128,1,float16,fp8,0,0.27162667115529376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,1,128,1,float16,fp8,0,0.23443732659022012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,2,128,1,float16,float16,0,0.24409067630767822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,2,128,1,float16,fp8,0,0.2419253389040629
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,4,128,1,float16,float16,0,0.2518293261528015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,8,128,1,float16,float16,0,0.2587413390477498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,24,8,128,1,float16,fp8,0,0.2762453357378642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,1,128,1,float16,float16,0,3.588831901550293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,1,128,1,float16,fp8,0,3.6460800170898438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,2,128,1,float16,float16,0,3.6601813634236655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,4,128,1,float16,float16,0,4.158032099405925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,4,128,1,float16,fp8,0,4.412218729654948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,8,128,1,float16,float16,0,4.201877276102702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,1,128,1,float16,float16,0,1.9052106539408367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,24,128,1,float16,float16,0,2.430389404296875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,8,128,1,float16,fp8,0,4.411823908487956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,24,2,128,1,float16,fp8,0,8.648895899454752
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,24,128,1,float16,fp8,0,2.432261308034261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,2,128,1,float16,float16,0,1.8608585993448894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,1,128,1,float16,fp8,0,1.9110612869262695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,2,128,1,float16,fp8,0,1.9664160410563152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,4,128,1,float16,float16,0,2.09824005762736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,4,128,1,float16,fp8,0,2.2261974016825357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,24,128,1,float16,float16,0,1.1645973523457844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,1,128,1,float16,float16,0,0.8958453337351481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,24,128,1,float16,fp8,0,1.2535946369171143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,1,128,1,float16,fp8,0,0.9533867041269938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,8,128,1,float16,fp8,0,2.220794677734375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,2,128,1,float16,float16,0,0.9498666922251383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,2,128,1,float16,fp8,0,0.9823626677195231
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,24,8,128,1,float16,float16,0,5.535919825236003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,4,128,1,float16,float16,0,1.059834639231364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,4,128,1,float16,fp8,0,1.1482880115509033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,8,128,1,float16,float16,0,1.0777173042297363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,24,128,1,float16,float16,0,0.6000639994939169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,24,8,128,1,float16,fp8,0,1.137386639912923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,24,128,1,float16,fp8,0,0.6483519872029623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,1,128,1,float16,fp8,0,0.4989653428395589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,1,128,1,float16,float16,0,0.8485546906789144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,2,128,1,float16,float16,0,0.4894133408864339
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,4,128,1,float16,float16,0,0.5466026862462362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,4,128,1,float16,fp8,0,0.592469334602356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,2,128,1,float16,fp8,0,0.5196373462677002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,8,128,1,float16,float16,0,0.552677313486735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,24,8,128,1,float16,fp8,0,0.6028319994608561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,24,128,1,float16,float16,0,0.3207520047823588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,1,128,1,float16,float16,0,0.24942400058110556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,24,128,1,float16,fp8,0,0.6284319957097372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,1,128,1,float16,fp8,0,0.2700586716334025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,2,128,1,float16,float16,0,0.2643360098203023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,2,128,1,float16,fp8,0,0.2805066704750061
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,4,128,1,float16,float16,0,0.28941865762074787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,4,128,1,float16,fp8,0,0.3196000059445699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,8,128,1,float16,float16,0,0.30140799283981323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,24,8,128,1,float16,fp8,0,0.32477333148320514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,24,128,1,float16,float16,0,0.19006933768590292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,4,128,1,float16,float16,0,0.16570666432380676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,24,128,1,float16,fp8,0,0.1978506644566854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,1,128,1,float16,float16,0,0.14312000075976053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,8,128,1,float16,fp8,0,0.18582399686177573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,1,128,1,float16,fp8,0,0.15034133195877075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,2,128,1,float16,float16,0,0.14898666739463806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,2,128,1,float16,fp8,0,0.15583999951680502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,4,128,1,float16,fp8,0,0.17683732509613037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,24,8,128,1,float16,float16,0,0.1717066764831543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,1,128,1,float16,float16,0,3.470010757446289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,1,128,1,float16,fp8,0,3.6637865702311196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,2,128,1,float16,float16,0,3.7391252517700195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,2,128,1,float16,fp8,0,3.7646506627400718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,4,128,1,float16,float16,0,4.266298611958821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,4,128,1,float16,fp8,0,4.613722801208496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,8,128,1,float16,float16,0,4.313610712687175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,24,128,1,float16,float16,0,2.430880069732666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,1,128,1,float16,float16,0,1.7232106526692708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,24,128,1,float16,fp8,0,2.5985546112060547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,2,128,1,float16,float16,0,1.8715039889017742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,1,128,1,float16,fp8,0,1.8283519744873047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,24,8,128,1,float16,fp8,0,8.052565256754557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,2,128,1,float16,fp8,0,1.8990613619486492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,4,128,1,float16,float16,0,2.1601972579956055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,8,128,1,float16,float16,0,2.168176015218099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,24,128,1,float16,fp8,0,1.31660262743632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,24,128,1,float16,float16,0,1.2324373722076416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,4,128,1,float16,fp8,0,2.389344056447347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,24,8,128,1,float16,fp8,0,2.312293370564779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,1,128,1,float16,float16,0,2.1584693590799966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,1,128,1,float16,fp8,0,0.9295893510182699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,2,128,1,float16,float16,0,0.9461226463317871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,8,128,1,float16,float16,0,1.1163787047068279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,2,128,1,float16,fp8,0,0.9870933691660563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,4,128,1,float16,float16,0,1.0702239672342937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,4,128,1,float16,fp8,0,1.2079040209452312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,24,128,1,float16,float16,0,0.648746649424235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,24,8,128,1,float16,fp8,0,1.2015413443247478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,24,128,1,float16,fp8,0,0.675818681716919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,1,128,1,float16,float16,0,0.4508533477783203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,1,128,1,float16,fp8,0,0.48947731653849286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,2,128,1,float16,float16,0,0.492298682530721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,2,128,1,float16,fp8,0,0.5065973202387491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,4,128,1,float16,float16,0,0.5656799872716268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,4,128,1,float16,fp8,0,0.6193439960479736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,8,128,1,float16,float16,0,0.5624746481577555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,24,8,128,1,float16,fp8,0,0.619269331296285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,24,128,1,float16,float16,0,0.3317546645800273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,24,128,1,float16,fp8,0,0.3619946638743083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,1,128,1,float16,float16,0,0.24172266324361166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,1,128,1,float16,fp8,0,0.26025599241256714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,2,128,1,float16,float16,0,0.2627573410669963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,2,128,1,float16,fp8,0,0.2721280058224996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,24,128,1,float16,fp8,0,0.19751999775568643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,4,128,1,float16,float16,0,0.29955732822418213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,4,128,1,float16,fp8,0,0.3221493363380432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,8,128,1,float16,float16,0,0.30562132596969604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,24,8,128,1,float16,fp8,0,0.3349119822184245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,24,128,1,float16,float16,0,0.1818186640739441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,4,128,1,float16,fp8,0,0.18034666776657104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,1,128,1,float16,float16,0,0.134250670671463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,1,128,1,float16,fp8,0,0.14774933457374573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,2,128,1,float16,float16,0,0.14381866653760275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,2,128,1,float16,fp8,0,0.15411200126012167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,4,128,1,float16,float16,0,0.1625440021355947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,1,128,1,float16,float16,0,0.07303999861081441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,8,128,1,float16,float16,0,0.16784000396728516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,24,8,128,1,float16,fp8,0,0.1844159960746765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,2,128,1,float16,fp8,0,0.08615466952323914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,24,128,1,float16,float16,0,0.10397332906723022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,24,128,1,float16,fp8,0,0.11666666467984517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,1,128,1,float16,fp8,0,0.08097599943478902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,2,128,1,float16,float16,0,0.07855999966462453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,4,128,1,float16,float16,0,0.08975999553998311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,4,128,1,float16,fp8,0,0.0953386624654134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,8,128,1,float16,float16,0,0.09589866797129314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,24,8,128,1,float16,fp8,0,0.1020906666914622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,1,128,1,float16,fp8,0,2.2824479738871255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,1,128,1,float16,float16,0,4.929845492045085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,2,128,1,float16,float16,0,2.2738773028055825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,2,128,1,float16,fp8,0,2.4259252548217773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,4,128,1,float16,float16,0,2.7412853240966797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,8,128,1,float16,float16,0,2.765712102254232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,24,128,1,float16,float16,0,1.5829866727193196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,24,128,1,float16,fp8,0,3.005605379740397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,1,128,1,float16,float16,0,1.0620960394541423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,8,128,1,float16,fp8,0,3.119098663330078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,1,128,1,float16,fp8,0,1.1555039882659912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,2,128,1,float16,fp8,0,1.2271253267923992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,2,128,1,float16,float16,0,1.1689493656158447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,24,4,128,1,float16,fp8,0,3.0418132146199546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,4,128,1,float16,float16,0,1.3795092900594075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,8,128,1,float16,float16,0,1.4058826764424641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,4,128,1,float16,fp8,0,2.8200372060139975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,24,128,1,float16,float16,0,0.8018293380737305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,24,8,128,1,float16,fp8,0,1.543349266052246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,24,128,1,float16,fp8,0,0.8953440189361572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,1,128,1,float16,float16,0,0.5479893287022909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,8,128,1,float16,float16,0,0.7186453342437744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,4,128,1,float16,fp8,0,0.7844533125559489
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,1,128,1,float16,fp8,0,0.5947733322779337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,2,128,1,float16,float16,0,0.607040007909139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,2,128,1,float16,fp8,0,0.6339519818623861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,1,128,1,float16,fp8,0,0.3134933312733968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,4,128,1,float16,float16,0,1.3828746477762859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,24,8,128,1,float16,fp8,0,0.8102453549702963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,2,128,1,float16,fp8,0,0.3290133277575175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,4,128,1,float16,fp8,0,0.42599467436472577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,24,128,1,float16,float16,0,0.41715200742085773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,24,128,1,float16,fp8,0,0.46326935291290283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,1,128,1,float16,float16,0,0.2877546747525533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,2,128,1,float16,float16,0,0.3187359968821208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,4,128,1,float16,float16,0,0.3708000183105469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,8,128,1,float16,float16,0,0.3744959831237793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,24,8,128,1,float16,fp8,0,0.418938676516215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,24,128,1,float16,float16,0,0.23715200026830038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,24,128,1,float16,fp8,0,0.2488106687863668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,1,128,1,float16,float16,0,0.158053328593572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,1,128,1,float16,fp8,0,0.17223467429478964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,2,128,1,float16,float16,0,0.17316800355911255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,24,128,1,float16,float16,0,0.12525866429011026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,2,128,1,float16,fp8,0,0.1811306675275167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,4,128,1,float16,float16,0,0.1986400087674459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,4,128,1,float16,fp8,0,0.21989333629608154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,8,128,1,float16,float16,0,0.2108479936917623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,24,8,128,1,float16,fp8,0,0.23034133513768515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,24,128,1,float16,fp8,0,0.1400053302447001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,1,128,1,float16,float16,0,0.08875733613967896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,1,128,1,float16,fp8,0,0.09552533427874248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,8,128,1,float16,float16,0,0.11389333009719849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,2,128,1,float16,float16,0,0.09755200147628784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,2,128,1,float16,fp8,0,0.10284800330797832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,4,128,1,float16,float16,0,0.10957866907119751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,4,128,1,float16,fp8,0,0.12476266423861186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,24,8,128,1,float16,fp8,0,0.12889066338539124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,24,128,1,float16,float16,0,0.07392533123493195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,24,128,1,float16,fp8,0,0.08362133304278056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,1,128,1,float16,float16,0,0.05348266661167145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,1,128,1,float16,fp8,0,0.06053866446018219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,8,128,1,float16,fp8,0,0.07165333131949107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,2,128,1,float16,float16,0,0.0536106675863266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,2,128,1,float16,fp8,0,0.06162666777769724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,4,128,1,float16,float16,0,0.06089066465695699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,4,128,1,float16,fp8,0,0.067221333583196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,24,8,128,1,float16,float16,0,0.06263466676076253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,1,128,1,float16,float16,0,2.200442632039388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,1,128,1,float16,fp8,0,2.4460585912068686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,2,128,1,float16,float16,0,2.4878133138020835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,2,128,1,float16,fp8,0,2.612858613332113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,24,128,1,float16,float16,0,1.7934986750284831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,8,128,1,float16,float16,0,3.0834506352742515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,4,128,1,float16,float16,0,3.017205238342285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,1,128,1,float16,float16,0,1.1233173211415608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,8,128,1,float16,fp8,0,3.5917174021402993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,24,4,128,1,float16,fp8,0,5.415482838948567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,24,128,1,float16,fp8,0,2.0467306772867837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,1,128,1,float16,fp8,0,1.247226635615031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,2,128,1,float16,fp8,0,1.3085760275522869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,2,128,1,float16,float16,0,1.2607893149058025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,4,128,1,float16,float16,0,1.5256800651550293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,8,128,1,float16,float16,0,1.5432319641113281
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,4,128,1,float16,fp8,0,1.7508427302042644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,24,128,1,float16,float16,0,0.9251413345336914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,1,128,1,float16,float16,0,0.5689599911371866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,24,128,1,float16,fp8,0,1.0460800329844158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,24,8,128,1,float16,fp8,0,2.638751983642578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,1,128,1,float16,fp8,0,0.632202665011088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,2,128,1,float16,fp8,0,0.6719733079274496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,2,128,1,float16,float16,0,0.6461280186971029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,4,128,1,float16,float16,0,0.7766719659169515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,8,128,1,float16,fp8,0,1.155717372894287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,4,128,1,float16,fp8,0,0.8762506643931071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,24,8,128,1,float16,float16,0,0.7897173563639323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,24,128,1,float16,float16,0,0.46305068333943683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,2,128,1,float16,float16,0,0.5025226672490438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,2,128,1,float16,fp8,0,0.35212798913319904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,24,128,1,float16,fp8,0,0.5326186815897623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,1,128,1,float16,float16,0,0.29496000210444134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,8,128,1,float16,float16,0,0.4574986696243286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,8,128,1,float16,fp8,0,0.46996267636617023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,4,128,1,float16,fp8,0,0.4570666551589966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,1,128,1,float16,fp8,0,0.33186666170756024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,24,4,128,1,float16,float16,0,0.411631981531779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,24,128,1,float16,fp8,0,0.2797706723213196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,24,128,1,float16,float16,0,0.24984532594680786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,1,128,1,float16,float16,0,0.1627946694691976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,1,128,1,float16,fp8,0,0.23082667589187622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,2,128,1,float16,float16,0,0.17867199579874674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,2,128,1,float16,fp8,0,0.19243200620015463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,4,128,1,float16,float16,0,0.2153173287709554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,4,128,1,float16,fp8,0,0.24314665794372559
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,8,128,1,float16,float16,0,0.22213866313298544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,24,8,128,1,float16,fp8,0,0.2520959973335266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,24,128,1,float16,float16,0,0.13501333196957907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,24,128,1,float16,fp8,0,0.1550879975159963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,8,128,1,float16,float16,0,0.12190399567286174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,1,128,1,float16,float16,0,0.09080533186594646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,1,128,1,float16,fp8,0,0.10382933417956035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,2,128,1,float16,float16,0,0.10060800115267436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,2,128,1,float16,fp8,0,0.11172800262769063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,4,128,1,float16,float16,0,0.11805867155392964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,1,128,1,float16,fp8,0,0.05710933109124502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,4,128,1,float16,fp8,0,0.13462400436401367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,2,128,1,float16,fp8,0,0.06208533545335134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,24,8,128,1,float16,fp8,0,0.13926399747530618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,24,128,1,float16,float16,0,0.07826133569081624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,24,128,1,float16,fp8,0,0.09105066458384196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,1,128,1,float16,float16,0,0.051311999559402466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,24,128,1,float16,float16,0,0.047226667404174805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,2,128,1,float16,float16,0,0.053616002202034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,4,128,1,float16,float16,0,0.06321600079536438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,4,128,1,float16,fp8,0,0.07123733560244243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,8,128,1,float16,float16,0,0.06949866811434428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,24,8,128,1,float16,fp8,0,0.07680533329645793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,4,128,1,float16,float16,0,0.03933866570393244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,4,128,1,float16,fp8,0,0.04516266783078512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,24,128,1,float16,fp8,0,0.05261866748332977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,1,128,1,float16,float16,0,0.033770665526390076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,1,128,1,float16,fp8,0,0.03860266755024592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,2,128,1,float16,float16,0,0.03495466709136963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,2,128,1,float16,fp8,0,0.03965866565704346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,8,128,1,float16,float16,0,0.03942933430274328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,24,8,128,1,float16,fp8,0,0.045642669002215065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,1,128,1,float16,float16,0,1.6574026743570964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,1,128,1,float16,fp8,0,1.8737600644429524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,2,128,1,float16,float16,0,1.9289973576863606
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,2,128,1,float16,fp8,0,2.0650240580240884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,24,128,1,float16,float16,0,1.5044053395589192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,4,128,1,float16,float16,0,2.472383975982666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,24,128,1,float16,fp8,0,1.834554672241211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,1,128,1,float16,float16,0,0.835973342259725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,1,128,1,float16,fp8,0,0.9504160086313883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,2,128,1,float16,float16,0,0.9750826358795166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,4,128,1,float16,fp8,0,2.8903732299804688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,2,128,1,float16,fp8,0,1.0376160144805908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,4,128,1,float16,float16,0,1.2528746922810872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,8,128,1,float16,float16,0,3.3713385264078775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,4,128,1,float16,fp8,0,1.4669814109802246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,24,8,128,1,float16,fp8,0,2.9087092081705728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,8,128,1,float16,float16,0,1.2637866338094075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,24,128,1,float16,fp8,0,0.9283200105031332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,1,128,1,float16,float16,0,0.4283626476923625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,24,8,128,1,float16,fp8,0,1.4775519371032715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,24,128,1,float16,float16,0,0.7633866469065348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,2,128,1,float16,float16,0,0.50219198067983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,4,128,1,float16,fp8,0,0.7451519966125488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,24,128,1,float16,float16,0,0.39240535100301105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,1,128,1,float16,fp8,0,0.4973653157552083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,1,128,1,float16,float16,0,0.22421866655349731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,4,128,1,float16,float16,0,0.6337759892145792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,8,128,1,float16,float16,0,0.6588853200276693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,24,128,1,float16,fp8,0,0.47700798511505127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,2,128,1,float16,fp8,0,0.5320959885915121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,24,8,128,1,float16,fp8,0,0.7893866697947184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,1,128,1,float16,fp8,0,0.2560426592826843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,4,128,1,float16,fp8,0,0.38729600111643475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,2,128,1,float16,float16,0,0.2658240000406901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,2,128,1,float16,fp8,0,0.2790880004564921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,24,128,1,float16,float16,0,0.20646933714548746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,4,128,1,float16,float16,0,0.329093337059021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,8,128,1,float16,float16,0,0.3368693192799886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,24,8,128,1,float16,fp8,0,0.40702398618062335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,24,128,1,float16,fp8,0,0.2505493362744649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,4,128,1,float16,float16,0,0.17361066738764444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,1,128,1,float16,float16,0,0.12321600317955017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,1,128,1,float16,fp8,0,0.1423360009988149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,8,128,1,float16,fp8,0,0.21701333920160928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,2,128,1,float16,float16,0,0.1404213309288025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,2,128,1,float16,fp8,0,0.1514133314291636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,4,128,1,float16,fp8,0,0.20498667160669962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,24,8,128,1,float16,float16,0,0.18011732896169028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,24,128,1,float16,float16,0,0.11385599772135417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,24,128,1,float16,fp8,0,0.13846932848294577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,1,128,1,float16,float16,0,0.07063999772071838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,4,128,1,float16,fp8,0,0.11380799611409505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,1,128,1,float16,fp8,0,0.08242666721343994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,2,128,1,float16,float16,0,0.07918933530648549
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,2,128,1,float16,fp8,0,0.0881706674893697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,4,128,1,float16,float16,0,0.09617599844932556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,8,128,1,float16,float16,0,0.09909866253534953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,24,8,128,1,float16,fp8,0,0.11957866946856181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,24,128,1,float16,float16,0,0.06690666576226552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,4,128,1,float16,float16,0,0.05009600023428599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,24,128,1,float16,fp8,0,0.07995200157165527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,1,128,1,float16,float16,0,0.03945599993069967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,1,128,1,float16,fp8,0,0.04565866788228353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,24,128,1,float16,float16,0,0.03832533210515976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,2,128,1,float16,float16,0,0.04227200150489807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,2,128,1,float16,fp8,0,0.048538664976755776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,4,128,1,float16,fp8,0,0.05670933425426483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,8,128,1,float16,float16,0,0.057461331288019814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,24,8,128,1,float16,fp8,0,0.06467733283837636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,24,128,1,float16,fp8,0,0.04504533112049103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,1,128,1,float16,float16,0,0.026149332523345947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,8,128,1,float16,float16,0,0.030752000709374745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,1,128,1,float16,fp8,0,0.029733332494894665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,2,128,1,float16,float16,0,0.028058665494124096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,24,128,1,float16,fp8,0,0.02769600103298823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,2,128,1,float16,fp8,0,0.03316266586383184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,1,128,1,float16,fp8,0,0.022106667359670002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,4,128,1,float16,float16,0,0.0301706666747729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,4,128,1,float16,fp8,0,0.03793066740036011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,24,8,128,1,float16,fp8,0,0.038005332152048744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,24,128,1,float16,float16,0,0.022090665996074677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,1,128,1,float16,float16,0,0.019359999646743137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,2,128,1,float16,float16,0,0.019786667078733444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,2,128,1,float16,fp8,0,0.022954667607943218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,4,128,1,float16,float16,0,0.021002667645613354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,4,128,1,float16,fp8,0,0.025989333788553875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,8,128,1,float16,float16,0,0.021130666136741638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,24,8,128,1,float16,fp8,0,0.02649066597223282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,1,128,1,float16,float16,0,0.698970635732015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,1,128,1,float16,fp8,0,0.8113546371459961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,2,128,1,float16,float16,0,0.8131146430969238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,2,128,1,float16,fp8,0,0.8902080059051514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,4,128,1,float16,float16,0,1.1237066586812336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,24,128,1,float16,float16,0,0.7022186915079752
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,24,128,1,float16,fp8,0,0.8160053094228109
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,4,128,1,float16,fp8,0,1.326464017232259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,1,128,1,float16,float16,0,0.3606826861699422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,2,128,1,float16,float16,0,0.43802666664123535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,2,128,1,float16,fp8,0,0.459173321723938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,1,128,1,float16,fp8,0,0.41679465770721436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,8,128,1,float16,float16,0,1.1351626714070637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,24,8,128,1,float16,fp8,0,1.3339625994364421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,4,128,1,float16,float16,0,0.5658933321634928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,4,128,1,float16,fp8,0,0.6730559666951498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,24,128,1,float16,float16,0,0.3617333173751831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,8,128,1,float16,float16,0,0.5946346521377563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,24,128,1,float16,fp8,0,0.41896533966064453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,24,8,128,1,float16,fp8,0,0.7111946741739908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,2,128,1,float16,float16,0,0.22944533824920654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,1,128,1,float16,float16,0,0.19128000736236572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,2,128,1,float16,fp8,0,0.24302933613459268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,1,128,1,float16,fp8,0,0.2209119995435079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,4,128,1,float16,float16,0,0.2919413248697917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,4,128,1,float16,fp8,0,0.35051735242207843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,8,128,1,float16,float16,0,0.3121066689491272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,24,8,128,1,float16,fp8,0,0.3651680151621501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,24,128,1,float16,float16,0,0.19062934319178262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,2,128,1,float16,fp8,0,0.1323253313700358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,24,128,1,float16,fp8,0,0.22317866484324136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,1,128,1,float16,float16,0,0.10429867108662923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,1,128,1,float16,fp8,0,0.12128000458081563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,2,128,1,float16,float16,0,0.12126400073369344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,4,128,1,float16,float16,0,0.1555519998073578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,4,128,1,float16,fp8,0,0.18659732739130655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,8,128,1,float16,fp8,0,0.1965706745783488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,24,8,128,1,float16,float16,0,0.16473066806793213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,24,128,1,float16,float16,0,0.10522133111953735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,24,128,1,float16,fp8,0,0.12261866529782613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,1,128,1,float16,float16,0,0.06084266801675161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,1,128,1,float16,fp8,0,0.07074666519959767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,2,128,1,float16,float16,0,0.06986666719118755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,2,128,1,float16,fp8,0,0.07784533500671387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,4,128,1,float16,float16,0,0.08574400345484416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,4,128,1,float16,fp8,0,0.1030346651871999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,8,128,1,float16,float16,0,0.09133866429328918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,24,8,128,1,float16,fp8,0,0.10832533240318298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,24,128,1,float16,float16,0,0.059845333298047386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,24,128,1,float16,fp8,0,0.07065066695213318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,1,128,1,float16,float16,0,0.03282133241494497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,4,128,1,float16,fp8,0,0.05016533533732096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,1,128,1,float16,fp8,0,0.03779733429352442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,2,128,1,float16,float16,0,0.03515200068553289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,2,128,1,float16,fp8,0,0.04107200105985006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,4,128,1,float16,float16,0,0.043141335248947144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,8,128,1,float16,float16,0,0.04937066634496053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,24,8,128,1,float16,fp8,0,0.05547733108202616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,2,128,1,float16,float16,0,0.023397333920001984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,24,128,1,float16,float16,0,0.03427733232577642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,24,128,1,float16,fp8,0,0.038015998899936676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,1,128,1,float16,float16,0,0.022463999688625336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,1,128,1,float16,fp8,0,0.026186667382717133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,2,128,1,float16,fp8,0,0.027471999327341717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,4,128,1,float16,float16,0,0.025920001169045765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,4,128,1,float16,fp8,0,0.03203733265399933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,8,128,1,float16,float16,0,0.02611733227968216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,24,8,128,1,float16,fp8,0,0.03236799935499827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,24,128,1,float16,float16,0,0.019893333315849304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,2,128,1,float16,fp8,0,0.019658666104078293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,24,128,1,float16,fp8,0,0.024351999163627625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,1,128,1,float16,float16,0,0.01643199970324834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,1,128,1,float16,fp8,0,0.019359999646743137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,2,128,1,float16,float16,0,0.01802666609485944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,4,128,1,float16,float16,0,0.01821333294113477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,1,128,1,float16,float16,0,0.016250666230916977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,4,128,1,float16,fp8,0,0.02239466706911723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,8,128,1,float16,float16,0,0.018592000007629395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,24,8,128,1,float16,fp8,0,0.022543999056021374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,24,128,1,float16,float16,0,0.017386666188637417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,24,128,1,float16,fp8,0,0.021690666675567627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,8,128,1,float16,float16,0,0.017055999487638474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,1,128,1,float16,fp8,0,0.0189280000825723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,2,128,1,float16,float16,0,0.01632000009218852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,2,128,1,float16,fp8,0,0.019146667172511418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,4,128,1,float16,float16,0,0.016751999656359356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,4,128,1,float16,fp8,0,0.02093333254257838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,24,8,128,1,float16,fp8,0,0.020618667205174763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,1,128,1,float16,float16,0,0.3444799979527791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,1,128,1,float16,fp8,0,0.3755520184834798
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,2,128,1,float16,float16,0,0.4131679932276408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,24,128,1,float16,float16,0,0.3473333517710368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,2,128,1,float16,fp8,0,0.41973865032196045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,4,128,1,float16,fp8,0,0.6318613290786743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,4,128,1,float16,float16,0,0.5484266678492228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,24,128,1,float16,fp8,0,0.38251201311747235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,1,128,1,float16,float16,0,0.1827039917310079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,8,128,1,float16,float16,0,0.5602506796518961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,24,8,128,1,float16,fp8,0,0.6651093165079752
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,1,128,1,float16,fp8,0,0.20037333170572916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,8,128,1,float16,float16,0,0.2912213404973348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,2,128,1,float16,float16,0,0.21546133359273276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,24,128,1,float16,float16,0,0.18288532892862955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,4,128,1,float16,float16,0,0.28220800558725995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,2,128,1,float16,fp8,0,0.22209066152572632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,1,128,1,float16,fp8,0,0.11553066968917847
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,2,128,1,float16,float16,0,0.11658666531244914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,24,128,1,float16,fp8,0,0.2029973268508911
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,4,128,1,float16,fp8,0,0.32756266991297406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,24,8,128,1,float16,fp8,0,0.34466667970021564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,1,128,1,float16,float16,0,0.102101335922877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,2,128,1,float16,fp8,0,0.1246613363424937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,4,128,1,float16,float16,0,0.14986667037010193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,4,128,1,float16,fp8,0,0.1760480006535848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,8,128,1,float16,float16,0,0.155157337586085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,24,8,128,1,float16,fp8,0,0.18238399426142374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,24,128,1,float16,float16,0,0.10133866469065349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,24,128,1,float16,fp8,0,0.11104533076286316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,1,128,1,float16,float16,0,0.05937600135803223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,1,128,1,float16,fp8,0,0.06542933483918507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,2,128,1,float16,float16,0,0.06809066732724507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,2,128,1,float16,fp8,0,0.07293333113193512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,4,128,1,float16,float16,0,0.08401067058245341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,4,128,1,float16,fp8,0,0.09557867050170898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,8,128,1,float16,float16,0,0.08594133456548055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,24,8,128,1,float16,fp8,0,0.09914132952690125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,24,128,1,float16,float16,0,0.057850668827692665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,24,128,1,float16,fp8,0,0.062218666076660156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,1,128,1,float16,float16,0,0.029178666571776073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,8,128,1,float16,float16,0,0.04866666595141093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,1,128,1,float16,fp8,0,0.03270933280388514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,2,128,1,float16,float16,0,0.031514666974544525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,2,128,1,float16,fp8,0,0.035530666510264076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,4,128,1,float16,float16,0,0.03965866565704346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,4,128,1,float16,fp8,0,0.044079999128977455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,24,8,128,1,float16,fp8,0,0.04670399924119314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,24,128,1,float16,float16,0,0.03276800115903219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,4,128,1,float16,float16,0,0.02481066683928172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,24,128,1,float16,fp8,0,0.03268799930810928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,1,128,1,float16,float16,0,0.020549333343903225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,1,128,1,float16,fp8,0,0.023872000475724537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,2,128,1,float16,float16,0,0.021717332303524017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,24,128,1,float16,fp8,0,0.020197333147128422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,2,128,1,float16,fp8,0,0.025450666745503742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,1,128,1,float16,fp8,0,0.01777600000301997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,4,128,1,float16,fp8,0,0.028607999285062153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,2,128,1,float16,float16,0,0.01543466622630755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,8,128,1,float16,float16,0,0.025429333249727886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,24,8,128,1,float16,fp8,0,0.028730665644009907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,24,128,1,float16,float16,0,0.017530667285124462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,1,128,1,float16,float16,0,0.015562667200962702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,2,128,1,float16,fp8,0,0.01775466650724411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,4,128,1,float16,float16,0,0.016719999412695568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,4,128,1,float16,fp8,0,0.019493332753578823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,8,128,1,float16,float16,0,0.016917333006858826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,24,8,128,1,float16,fp8,0,0.019509332875410717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,24,128,1,float16,float16,0,0.015520000209410986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,24,128,1,float16,fp8,0,0.017727999637524288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,1,128,1,float16,float16,0,0.014629332969586054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,1,128,1,float16,fp8,0,0.017184000462293625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,2,128,1,float16,float16,0,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,2,128,1,float16,fp8,0,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,4,128,1,float16,float16,0,0.015370666980743408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,4,128,1,float16,fp8,0,0.0179626668492953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,8,128,1,float16,float16,0,0.01526933287580808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,24,8,128,1,float16,fp8,0,0.017477333545684814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,24,128,1,float16,float16,0,0.014885333677132925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,24,128,1,float16,fp8,0,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,1,128,1,float16,float16,0,0.0145066666106383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,1,128,1,float16,fp8,0,0.017194667210181553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,2,128,1,float16,float16,0,0.01479999969402949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,2,128,1,float16,fp8,0,0.016949333250522614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,4,128,1,float16,float16,0,0.014645333091417948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,4,128,1,float16,fp8,0,0.01728533332546552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,8,128,1,float16,float16,0,0.014949332922697067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,24,8,128,1,float16,fp8,0,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,1,128,1,float16,fp8,0,0.19935999313990274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,2,128,1,float16,float16,0,0.2169439991315206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,2,128,1,float16,fp8,0,0.22244266668955484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,4,128,1,float16,float16,0,0.28294400374094647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,4,128,1,float16,fp8,0,0.32627199093500775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,1,128,1,float16,float16,0,0.18103466431299844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,8,128,1,float16,float16,0,0.32425065835316974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,1,128,1,float16,fp8,0,0.11422399679819743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,24,128,1,float16,float16,0,0.20099733273188272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,24,8,128,1,float16,fp8,0,0.3686399857203166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,24,128,1,float16,fp8,0,0.20285866657892862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,1,128,1,float16,float16,0,0.09889066219329834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,2,128,1,float16,float16,0,0.11752532919247945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,2,128,1,float16,fp8,0,0.1251146694024404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,4,128,1,float16,float16,0,0.15035200119018555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,4,128,1,float16,fp8,0,0.17557867368062338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,8,128,1,float16,float16,0,0.17298134167989096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,24,128,1,float16,float16,0,0.10858666896820068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,24,8,128,1,float16,fp8,0,0.202239990234375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,2,128,1,float16,fp8,0,0.0722453345855077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,24,128,1,float16,fp8,0,0.10928533474604289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,1,128,1,float16,float16,0,0.05820266902446747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,1,128,1,float16,fp8,0,0.06465599934260051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,2,128,1,float16,float16,0,0.0679253339767456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,4,128,1,float16,float16,0,0.08453866839408875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,4,128,1,float16,fp8,0,0.09707732995351155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,8,128,1,float16,float16,0,0.09515200058619182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,24,8,128,1,float16,fp8,0,0.10884267091751099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,24,128,1,float16,float16,0,0.06076799829800924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,24,128,1,float16,fp8,0,0.058373332023620605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,1,128,1,float16,float16,0,0.028501334289709728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,1,128,1,float16,fp8,0,0.03238933285077413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,2,128,1,float16,float16,0,0.03169599920511246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,2,128,1,float16,fp8,0,0.03562666724125544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,4,128,1,float16,float16,0,0.038319999972979225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,4,128,1,float16,fp8,0,0.04268266757329305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,8,128,1,float16,float16,0,0.050330668687820435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,24,8,128,1,float16,fp8,0,0.05329066514968872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,24,128,1,float16,float16,0,0.031189332405726116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,24,128,1,float16,fp8,0,0.02779199928045273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,1,128,1,float16,float16,0,0.021162666380405426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,1,128,1,float16,fp8,0,0.02403733382622401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,2,128,1,float16,float16,0,0.021530665457248688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,2,128,1,float16,fp8,0,0.025392000873883564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,4,128,1,float16,float16,0,0.024847999215126038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,4,128,1,float16,fp8,0,0.028490667541821797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,8,128,1,float16,float16,0,0.024661332368850708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,24,8,128,1,float16,fp8,0,0.02855466554562251
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,2,128,1,float16,fp8,0,0.017605333278576534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,24,128,1,float16,float16,0,0.019280000279347103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,24,128,1,float16,fp8,0,0.019343999524911244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,1,128,1,float16,float16,0,0.014906667172908783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,1,128,1,float16,fp8,0,0.01809599995613098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,2,128,1,float16,float16,0,0.015706667055686314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,4,128,1,float16,float16,0,0.01685333376129468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,1,128,1,float16,float16,0,0.014560000350077948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,4,128,1,float16,fp8,0,0.019472000499566395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,8,128,1,float16,float16,0,0.016501333564519882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,24,8,128,1,float16,fp8,0,0.01942933350801468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,24,128,1,float16,float16,0,0.013776000589132309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,24,128,1,float16,fp8,0,0.014117332796255747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,8,128,1,float16,float16,0,0.015008000036080679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,1,128,1,float16,fp8,0,0.017263999829689663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,2,128,1,float16,float16,0,0.014970666418472925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,2,128,1,float16,fp8,0,0.017397332936525345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,4,128,1,float16,float16,0,0.015210667004187902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,4,128,1,float16,fp8,0,0.017664000391960144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,24,8,128,1,float16,fp8,0,0.01724799970785777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,24,128,1,float16,float16,0,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,24,128,1,float16,fp8,0,0.013530666629473368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,1,128,1,float16,float16,0,0.014698666830857595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,1,128,1,float16,fp8,0,0.016917333006858826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,2,128,1,float16,float16,0,0.01458666721979777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,24,128,1,float16,float16,0,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,2,128,1,float16,fp8,0,0.01714133347074191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,4,128,1,float16,float16,0,0.014479999740918478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,4,128,1,float16,fp8,0,0.017279999951521557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,8,128,1,float16,float16,0,0.014842666685581207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,24,8,128,1,float16,fp8,0,0.017562666287024815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,2,128,1,float16,fp8,0,0.01581866666674614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,24,128,1,float16,fp8,0,0.013066666821638743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,1,128,1,float16,float16,0,0.013754667093356451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,1,128,1,float16,fp8,0,0.016255999604860943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,2,128,1,float16,float16,0,0.01360000049074491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,4,128,1,float16,float16,0,0.014282666146755219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,4,128,1,float16,fp8,0,0.01598400001724561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,8,128,1,float16,float16,0,0.014005333185195923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,24,8,128,1,float16,fp8,0,0.016122666498025257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,1,128,1,float16,float16,0,0.09875733653704326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,1,128,1,float16,fp8,0,0.11362666885058086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,2,128,1,float16,float16,0,0.11578133702278137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,2,128,1,float16,fp8,0,0.12587733070055643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,4,128,1,float16,float16,0,0.16664000352223715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,24,128,1,float16,float16,0,0.14383467038472494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,4,128,1,float16,fp8,0,0.1944213310877482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,8,128,1,float16,fp8,0,0.20042133331298828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,24,128,1,float16,fp8,0,0.14124266306559244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,1,128,1,float16,float16,0,0.05841066439946493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,1,128,1,float16,fp8,0,0.06578133503595988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,2,128,1,float16,float16,0,0.06751466790835063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,24,8,128,1,float16,float16,0,0.17182934284210205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,2,128,1,float16,fp8,0,0.07317333420117696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,4,128,1,float16,float16,0,0.09230400125185649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,4,128,1,float16,fp8,0,0.10634666681289673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,8,128,1,float16,float16,0,0.09883200128873189
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,24,8,128,1,float16,fp8,0,0.10910399754842122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,2,128,1,float16,fp8,0,0.03532800078392029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,24,128,1,float16,float16,0,0.07869333525498708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,24,128,1,float16,fp8,0,0.07769066592057546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,1,128,1,float16,float16,0,0.02847466617822647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,1,128,1,float16,fp8,0,0.03253866732120514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,2,128,1,float16,float16,0,0.031530665854612984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,4,128,1,float16,float16,0,0.04317333300908407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,4,128,1,float16,fp8,0,0.04846400022506714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,1,128,1,float16,fp8,0,0.023904000719388325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,8,128,1,float16,float16,0,0.051301335295041404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,24,8,128,1,float16,fp8,0,0.05354666709899902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,4,128,1,float16,float16,0,0.024143998821576435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,24,128,1,float16,float16,0,0.040693332751592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,24,128,1,float16,fp8,0,0.03739733248949051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,1,128,1,float16,float16,0,0.020773333807786305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,2,128,1,float16,float16,0,0.022229333718617756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,2,128,1,float16,fp8,0,0.025194667279720306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,4,128,1,float16,fp8,0,0.028431999186674755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,8,128,1,float16,float16,0,0.024256000916163128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,24,8,128,1,float16,fp8,0,0.0286613330245018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,24,128,1,float16,float16,0,0.02204799900452296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,24,128,1,float16,fp8,0,0.02250666668017705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,1,128,1,float16,float16,0,0.015520000209410986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,1,128,1,float16,fp8,0,0.01794133335351944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,2,128,1,float16,float16,0,0.015322666615247726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,2,128,1,float16,fp8,0,0.017877332866191864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,4,128,1,float16,float16,0,0.016362667083740234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,4,128,1,float16,fp8,0,0.019343999524911244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,8,128,1,float16,float16,0,0.01655999943614006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,24,8,128,1,float16,fp8,0,0.01915733392039935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,24,128,1,float16,float16,0,0.016682667036851246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,24,128,1,float16,fp8,0,0.016858667135238647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,4,128,1,float16,fp8,0,0.01717866708834966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,1,128,1,float16,float16,0,0.014655999839305878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,1,128,1,float16,fp8,0,0.017269333203633625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,2,128,1,float16,float16,0,0.014485333114862442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,2,128,1,float16,fp8,0,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,4,128,1,float16,float16,0,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,8,128,1,float16,float16,0,0.014815999815861383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,24,8,128,1,float16,fp8,0,0.017477333545684814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,2,128,1,float16,fp8,0,0.017045332739750545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,24,128,1,float16,float16,0,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,24,128,1,float16,fp8,0,0.013557333499193192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,1,128,1,float16,float16,0,0.014570667097965876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,1,128,1,float16,fp8,0,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,2,128,1,float16,float16,0,0.01452800010641416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,4,128,1,float16,float16,0,0.014570667097965876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,4,128,1,float16,fp8,0,0.017194667210181553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,1,128,1,float16,fp8,0,0.01639466608564059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,8,128,1,float16,float16,0,0.014698666830857595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,24,8,128,1,float16,fp8,0,0.016751999656359356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,4,128,1,float16,float16,0,0.014549333602190018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,24,128,1,float16,float16,0,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,24,128,1,float16,fp8,0,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,1,128,1,float16,float16,0,0.0136266661187013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,2,128,1,float16,float16,0,0.014175999909639359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,2,128,1,float16,fp8,0,0.015791999797026317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,1,128,1,float16,float16,0,0.013429333766301474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,4,128,1,float16,fp8,0,0.015791999797026317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,8,128,1,float16,float16,0,0.014352000008026758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,24,8,128,1,float16,fp8,0,0.01703466723362605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,24,128,1,float16,float16,0,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,24,128,1,float16,fp8,0,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,1,128,1,float16,fp8,0,0.015466666469971338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,2,128,1,float16,float16,0,0.013552000125249227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,2,128,1,float16,fp8,0,0.01533866673707962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,4,128,1,float16,float16,0,0.013781332721312841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,4,128,1,float16,fp8,0,0.01581866666674614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,8,128,1,float16,float16,0,0.013754667093356451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,24,8,128,1,float16,fp8,0,0.01544533297419548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,1,128,1,float16,float16,0,0.05716800192991892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,1,128,1,float16,fp8,0,0.06487466891606648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,2,128,1,float16,float16,0,0.07595199843247731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,4,128,1,float16,float16,0,0.09356266260147095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,2,128,1,float16,fp8,0,0.08358933528264363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,4,128,1,float16,fp8,0,0.10647466778755188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,8,128,1,float16,float16,0,0.10525866349538167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,24,128,1,float16,float16,0,0.11653866370519002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,24,8,128,1,float16,fp8,0,0.10683733224868774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,24,128,1,float16,fp8,0,0.11347200473149617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,1,128,1,float16,float16,0,0.028736000259717304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,1,128,1,float16,fp8,0,0.03266133368015289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,2,128,1,float16,float16,0,0.03594133257865906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,2,128,1,float16,fp8,0,0.04025600105524063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,4,128,1,float16,float16,0,0.04487466812133789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,4,128,1,float16,fp8,0,0.047653332352638245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,8,128,1,float16,float16,0,0.060496002435684204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,24,8,128,1,float16,fp8,0,0.05020266771316528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,24,128,1,float16,float16,0,0.061146666606267296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,24,128,1,float16,fp8,0,0.05705599983533224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,1,128,1,float16,float16,0,0.020474666108687718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,1,128,1,float16,fp8,0,0.02370133250951767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,2,128,1,float16,float16,0,0.021850667893886566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,2,128,1,float16,fp8,0,0.025029333929220837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,4,128,1,float16,float16,0,0.0242399995525678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,4,128,1,float16,fp8,0,0.028351999819278717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,8,128,1,float16,float16,0,0.027429332335789997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,24,8,128,1,float16,fp8,0,0.028560000161329906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,2,128,1,float16,fp8,0,0.017477333545684814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,24,128,1,float16,float16,0,0.031445334355036415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,24,128,1,float16,fp8,0,0.031930667658646904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,1,128,1,float16,float16,0,0.015466666469971338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,1,128,1,float16,fp8,0,0.017664000391960144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,2,128,1,float16,float16,0,0.015263999501864115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,4,128,1,float16,float16,0,0.016586666305859882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,4,128,1,float16,fp8,0,0.019343999524911244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,8,128,1,float16,float16,0,0.0179626668492953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,24,8,128,1,float16,fp8,0,0.018672000616788864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,24,128,1,float16,float16,0,0.020186666399240494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,24,128,1,float16,fp8,0,0.020784000555674236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,1,128,1,float16,float16,0,0.014485333114862442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,8,128,1,float16,float16,0,0.013157332936922709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,1,128,1,float16,fp8,0,0.01766933376590411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,2,128,1,float16,float16,0,0.014698666830857595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,2,128,1,float16,fp8,0,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,4,128,1,float16,float16,0,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,4,128,1,float16,fp8,0,0.016976000120242436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,2,128,1,float16,float16,0,0.014474666366974512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,2,128,1,float16,fp8,0,0.016783999900023144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,24,8,128,1,float16,fp8,0,0.013610667238632837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,24,128,1,float16,float16,0,0.016250666230916977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,8,128,1,float16,float16,0,0.012810666114091873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,24,128,1,float16,fp8,0,0.016719999412695568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,1,128,1,float16,float16,0,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,1,128,1,float16,fp8,0,0.017263999829689663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,4,128,1,float16,float16,0,0.014197333405415217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,4,128,1,float16,fp8,0,0.018789333601792652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,24,8,128,1,float16,fp8,0,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,24,128,1,float16,fp8,0,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,24,128,1,float16,float16,0,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,4,128,1,float16,fp8,0,0.016757333030303318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,8,128,1,float16,float16,0,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,1,128,1,float16,float16,0,0.014042666802803675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,1,128,1,float16,fp8,0,0.015642666568358738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,2,128,1,float16,float16,0,0.013904000322024027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,2,128,1,float16,fp8,0,0.015717333803574245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,4,128,1,float16,float16,0,0.01360000049074491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,24,8,128,1,float16,fp8,0,0.013007999708255133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,24,128,1,float16,float16,0,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,24,128,1,float16,fp8,0,0.01292266696691513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,4,128,1,float16,fp8,0,0.015583999454975128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,8,128,1,float16,float16,0,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,1,128,1,float16,float16,0,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,1,128,1,float16,fp8,0,0.015429332852363586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,2,128,1,float16,float16,0,0.013679999858140945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,1,128,1,float16,float16,0,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,2,128,1,float16,fp8,0,0.015482666591803232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,4,128,1,float16,float16,0,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,24,8,128,1,float16,fp8,0,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,4,128,1,float16,float16,0,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,24,128,1,float16,float16,0,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,24,128,1,float16,fp8,0,0.013050666699806849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,1,128,1,float16,fp8,0,0.015370666980743408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,2,128,1,float16,float16,0,0.01358933374285698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,2,128,1,float16,fp8,0,0.015263999501864115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,4,128,1,float16,fp8,0,0.015354666858911514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,8,128,1,float16,float16,0,0.011850666254758835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,24,8,128,1,float16,fp8,0,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,0,0.020351999749739964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,0,0.022069332500298817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,0,0.026421333352724712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,0,0.028789333999156952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,0,0.0397119993964831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,24,128,1,float16,fp8,0,0.09315733114878337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,0,0.042378668983777366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,0,0.06508266429106395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,24,128,1,float16,float16,0,0.08764266967773438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,0,0.0691786656777064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,0,0.013893333574136099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,0,0.015354666858911514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,0,0.017514667163292568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,0,0.019882666567961376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,0,0.023957334458827972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,0,0.025829332570234936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,0,0.03682666768630346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,0,0.039594667653242745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,24,128,1,float16,float16,0,0.049653331438700356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,24,128,1,float16,fp8,0,0.052426666021347046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,0,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,0,0.013503999759753546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,0,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,0,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,0,0.016719999412695568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,0,0.01777600000301997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,0,0.02387733260790507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,0,0.025279998779296875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,24,128,1,float16,float16,0,0.03012799968322118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,24,128,1,float16,fp8,0,0.031925333042939506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,0,0.011802667131026586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,0,0.012634667257467905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,0,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,0,0.012565333396196365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,0,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,0,0.013343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,0,0.016271999726692837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,0,0.017386666188637417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,24,128,1,float16,float16,0,0.019760000209013622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,0,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,24,128,1,float16,fp8,0,0.021242665747801464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,0,0.013045333325862885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,0,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,0,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,0,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,0,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,0,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,0,0.013173333058754602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,24,128,1,float16,float16,0,0.015573333948850632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,24,128,1,float16,fp8,0,0.016783999900023144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,0,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,0,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,0,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,0,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,0,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,0,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,0,0.011343999455372492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,0,0.011685332904259363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,0,0.012789333860079447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,24,128,1,float16,float16,0,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,24,128,1,float16,fp8,0,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,0,0.01091733326514562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,0,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,0,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,0,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,0,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,0,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,0,0.012954667210578918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,24,128,1,float16,float16,0,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,24,128,1,float16,fp8,0,0.013104000439246496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,0,0.010944000134865442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,0,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,0,0.011168000598748526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,0,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,0,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,0,0.011936000237862269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,0,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,0,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,24,128,1,float16,float16,0,0.011701333026091257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,24,128,1,float16,fp8,0,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,0,0.010944000134865442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,0,0.011920000116030375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,0,0.010885333021481832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,0,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,0,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,0,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,0,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,0,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,1,128,1,float16,fp8,0,13.770511627197266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,1,128,1,float16,float16,0,15.935877482096354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,2,128,1,float16,fp8,0,14.012917836507162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,2,128,1,float16,float16,0,15.745269775390625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,4,128,1,float16,fp8,0,14.673627217610678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,8,128,1,float16,float16,0,15.95681635538737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,8,128,1,float16,fp8,0,14.625279744466146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,16,128,1,float16,float16,0,8.599103927612305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,16,128,1,float16,fp8,0,7.519861221313477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,1,128,1,float16,float16,0,7.46234130859375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,1,128,1,float16,fp8,0,6.987791697184245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,2,128,1,float16,float16,0,7.265125274658203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,2,128,1,float16,fp8,0,7.073829650878906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,16,128,1,float16,float16,0,3.9632959365844727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,4,128,1,float16,float16,0,8.208607991536459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,4,128,1,float16,fp8,0,7.571290969848633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,8,128,1,float16,float16,0,7.961850484212239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,16,128,1,float16,fp8,0,4.290906588236491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,16,8,128,1,float16,fp8,0,7.496480305989583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,1,128,1,float16,float16,0,3.5701173146565757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,1,128,1,float16,fp8,0,3.6852426528930664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,2,128,1,float16,float16,0,3.6128905614217124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,16,4,128,1,float16,float16,0,38.979339599609375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,2,128,1,float16,fp8,0,3.5985708236694336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,4,128,1,float16,float16,0,3.8335679372151694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,4,128,1,float16,fp8,0,3.779605229695638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,8,128,1,float16,float16,0,3.8048585255940757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,16,128,1,float16,float16,0,2.031269391377767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,1,128,1,float16,float16,0,1.8282453219095867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,16,8,128,1,float16,fp8,0,3.875856081644694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,1,128,1,float16,fp8,0,1.9478507041931152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,2,128,1,float16,float16,0,1.7950506210327148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,16,128,1,float16,fp8,0,3.5886611938476562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,2,128,1,float16,fp8,0,1.9925600687662761
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,4,128,1,float16,float16,0,1.9591253598531086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,4,128,1,float16,fp8,0,1.9544906616210938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,8,128,1,float16,float16,0,1.959935983022054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,16,8,128,1,float16,fp8,0,2.0584053993225098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,1,128,1,float16,float16,0,8.790341059366861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,2,128,1,float16,float16,0,8.971413294474283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,2,128,1,float16,fp8,0,8.219055811564127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,4,128,1,float16,float16,0,9.62282689412435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,8,128,1,float16,fp8,0,8.732042948404947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,8,128,1,float16,float16,0,9.517386754353842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,16,128,1,float16,float16,0,4.696277300516765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,16,128,1,float16,fp8,0,4.66213321685791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,4,128,1,float16,fp8,0,8.809647878011068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,1,128,1,float16,float16,0,4.23362668355306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,16,1,128,1,float16,fp8,0,17.727291107177734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,2,128,1,float16,float16,0,4.104202588399251
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,1,128,1,float16,fp8,0,4.2673492431640625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,2,128,1,float16,fp8,0,4.22865613301595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,4,128,1,float16,float16,0,4.499264081319173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,4,128,1,float16,fp8,0,4.369365374247233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,1,128,1,float16,float16,0,2.0661919911702475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,16,128,1,float16,fp8,0,2.6197813351949057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,8,128,1,float16,float16,0,4.409834543863933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,1,128,1,float16,fp8,0,2.056154727935791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,2,128,1,float16,float16,0,2.0492480595906577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,16,8,128,1,float16,fp8,0,4.676618576049805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,16,128,1,float16,float16,0,7.400576273600261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,2,128,1,float16,fp8,0,2.096826712290446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,4,128,1,float16,float16,0,2.2695306142171225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,4,128,1,float16,fp8,0,2.2787307103474936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,8,128,1,float16,float16,0,2.2651519775390625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,16,128,1,float16,float16,0,1.216218630472819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,16,8,128,1,float16,fp8,0,2.258277257283529
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,16,128,1,float16,fp8,0,1.231760025024414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,1,128,1,float16,fp8,0,1.0754666328430176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,1,128,1,float16,float16,0,2.604303995768229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,8,128,1,float16,float16,0,1.166053295135498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,2,128,1,float16,float16,0,1.0717706680297852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,2,128,1,float16,fp8,0,1.1007839838663738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,4,128,1,float16,float16,0,1.1541439692179363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,4,128,1,float16,fp8,0,1.2058293024698894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,16,8,128,1,float16,fp8,0,1.2088692982991536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,1,128,1,float16,float16,0,6.034586588541667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,2,128,1,float16,float16,0,5.888010660807292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,2,128,1,float16,fp8,0,5.751701354980469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,4,128,1,float16,float16,0,6.4398345947265625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,4,128,1,float16,fp8,0,6.222906748453776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,8,128,1,float16,float16,0,6.217482884724935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,1,128,1,float16,fp8,0,8.770282745361328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,16,8,128,1,float16,fp8,0,6.297098795572917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,16,128,1,float16,float16,0,3.42525323232015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,1,128,1,float16,float16,0,2.869317372639974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,16,128,1,float16,fp8,0,3.4908320109049478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,1,128,1,float16,fp8,0,3.062922795613607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,2,128,1,float16,float16,0,2.8878933588663735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,2,128,1,float16,fp8,0,3.196314811706543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,4,128,1,float16,fp8,0,3.146538734436035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,8,128,1,float16,float16,0,3.2738720575968423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,16,128,1,float16,fp8,0,1.7831093470255535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,16,128,1,float16,float16,0,1.6755253473917644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,8,128,1,float16,fp8,0,3.5643946329752603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,1,128,1,float16,float16,0,1.4827359517415364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,1,128,1,float16,fp8,0,1.4739573796590169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,16,4,128,1,float16,float16,0,7.580389022827148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,2,128,1,float16,float16,0,1.4614720344543457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,2,128,1,float16,fp8,0,1.5018773078918457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,4,128,1,float16,float16,0,1.60098663965861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,4,128,1,float16,fp8,0,1.6544267336527507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,8,128,1,float16,float16,0,1.627893288930257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,16,128,1,float16,float16,0,0.8620906670888265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,16,8,128,1,float16,fp8,0,1.7033920288085938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,16,128,1,float16,fp8,0,1.0298720200856526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,1,128,1,float16,float16,0,0.7600800196329752
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,1,128,1,float16,fp8,0,0.7760319709777832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,2,128,1,float16,float16,0,0.7686986923217773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,2,128,1,float16,fp8,0,0.8109172979990641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,4,128,1,float16,float16,0,0.8372906843821207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,4,128,1,float16,fp8,0,0.8540799617767334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,8,128,1,float16,float16,0,0.8469066619873047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,16,8,128,1,float16,fp8,0,4.409328142801921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,1,128,1,float16,float16,0,7.623797098795573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,1,128,1,float16,fp8,0,7.310255686442058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,2,128,1,float16,float16,0,7.9488264719645185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,2,128,1,float16,fp8,0,7.477301279703776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,4,128,1,float16,float16,0,8.710997263590494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,4,128,1,float16,fp8,0,8.250298817952475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,8,128,1,float16,float16,0,8.958549499511719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,16,128,1,float16,float16,0,4.368831952412923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,16,128,1,float16,fp8,0,4.230143864949544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,1,128,1,float16,fp8,0,3.6915467580159507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,1,128,1,float16,float16,0,3.9832480748494468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,2,128,1,float16,float16,0,3.7851041158040366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,2,128,1,float16,fp8,0,4.150383949279785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,4,128,1,float16,float16,0,4.199973424275716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,4,128,1,float16,fp8,0,4.125631968180339
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,16,128,1,float16,float16,0,2.2160693804423013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,8,128,1,float16,float16,0,4.233210563659668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,16,8,128,1,float16,fp8,0,4.312549273173015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,16,128,1,float16,fp8,0,2.1773759524027505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,16,8,128,1,float16,fp8,0,12.228864034016928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,1,128,1,float16,float16,0,1.8956212997436523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,1,128,1,float16,fp8,0,1.864789326985677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,2,128,1,float16,float16,0,1.9592533111572266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,2,128,1,float16,fp8,0,1.9864266713460286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,4,128,1,float16,float16,0,2.094677289326986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,16,128,1,float16,float16,0,1.1307520071665447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,16,128,1,float16,fp8,0,1.1384212970733643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,4,128,1,float16,fp8,0,2.1237600644429526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,1,128,1,float16,float16,0,0.9783733685811361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,1,128,1,float16,fp8,0,1.014917294184367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,8,128,1,float16,float16,0,3.9609813690185547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,2,128,1,float16,float16,0,0.9877440134684244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,4,128,1,float16,fp8,0,1.0982027053833008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,16,8,128,1,float16,fp8,0,2.165210723876953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,2,128,1,float16,fp8,0,1.029077370961507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,4,128,1,float16,float16,0,1.0612160364786785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,8,128,1,float16,float16,0,1.0958773295084636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,16,8,128,1,float16,fp8,0,1.156394640604655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,16,128,1,float16,float16,0,0.6742719809214274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,16,128,1,float16,fp8,0,0.6292533477147421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,1,128,1,float16,float16,0,0.503712018330892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,1,128,1,float16,fp8,0,0.525871992111206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,8,128,1,float16,float16,0,0.5962026516596476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,2,128,1,float16,float16,0,0.516538659731547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,2,128,1,float16,fp8,0,0.5345600048700968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,4,128,1,float16,float16,0,0.5639199813206991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,4,128,1,float16,fp8,0,0.5885440111160278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,16,8,128,1,float16,fp8,0,0.6139626502990723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,1,128,1,float16,float16,0,4.399013201395671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,1,128,1,float16,fp8,0,4.287930806477864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,2,128,1,float16,fp8,0,4.468090693155925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,4,128,1,float16,float16,0,4.971578598022461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,4,128,1,float16,fp8,0,5.027498563130696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,16,128,1,float16,float16,0,2.6858507792154946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,8,128,1,float16,float16,0,5.095434824625651
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,8,128,1,float16,fp8,0,5.123786608378093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,16,128,1,float16,fp8,0,2.6110080083211265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,1,128,1,float16,float16,0,2.155504067738851
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,16,2,128,1,float16,float16,0,11.018384297688803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,1,128,1,float16,fp8,0,2.3380586306254068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,2,128,1,float16,float16,0,2.197663942972819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,2,128,1,float16,fp8,0,2.3570399284362793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,4,128,1,float16,float16,0,2.5087199211120605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,4,128,1,float16,fp8,0,2.5430240631103516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,8,128,1,float16,float16,0,2.472618738810221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,16,128,1,float16,fp8,0,1.3823787371317546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,16,8,128,1,float16,fp8,0,2.7326720555623374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,1,128,1,float16,float16,0,1.169141372044881
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,16,128,1,float16,float16,0,2.9035412470499673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,1,128,1,float16,fp8,0,1.1192106405893962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,2,128,1,float16,float16,0,1.1231839656829834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,2,128,1,float16,fp8,0,1.150597333908081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,4,128,1,float16,float16,0,1.2555519739786785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,4,128,1,float16,fp8,0,1.3021173477172852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,8,128,1,float16,float16,0,1.2782986958821614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,16,128,1,float16,float16,0,0.7716480096181234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,16,8,128,1,float16,fp8,0,1.3649546305338542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,16,128,1,float16,fp8,0,0.7136053244272867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,1,128,1,float16,float16,0,0.5735573371251425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,1,128,1,float16,fp8,0,0.5894879897435507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,2,128,1,float16,float16,0,0.5837013324101766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,2,128,1,float16,fp8,0,0.6060053507486979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,4,128,1,float16,float16,0,0.6557919979095459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,4,128,1,float16,fp8,0,0.9882559776306152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,8,128,1,float16,float16,0,0.6670186519622803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,16,8,128,1,float16,fp8,0,0.6976160208384196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,16,128,1,float16,float16,0,0.38209601243336994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,16,128,1,float16,fp8,0,0.3928266763687134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,4,128,1,float16,float16,0,0.35415999094645184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,4,128,1,float16,fp8,0,0.37429332733154297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,1,128,1,float16,float16,0,0.3113066752751668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,1,128,1,float16,fp8,0,0.3235519925753276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,2,128,1,float16,float16,0,0.31514666477839154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,2,128,1,float16,fp8,0,0.35676801204681396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,8,128,1,float16,float16,0,0.36611199378967285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,16,8,128,1,float16,fp8,0,0.38461867968241376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,1,128,1,float16,float16,0,4.0895999272664385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,1,128,1,float16,fp8,0,4.092293421427409
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,2,128,1,float16,float16,0,4.186949412027995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,2,128,1,float16,fp8,0,4.195263862609863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,4,128,1,float16,float16,0,4.970074653625488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,8,128,1,float16,float16,0,4.945173263549805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,16,128,1,float16,float16,0,2.6415732701619468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,1,128,1,float16,float16,0,2.0752693812052407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,8,128,1,float16,fp8,0,4.995631853739421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,16,128,1,float16,fp8,0,2.6479786237080893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,1,128,1,float16,fp8,0,2.0677760442097983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,16,4,128,1,float16,fp8,0,7.624575932820638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,2,128,1,float16,float16,0,2.085301399230957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,2,128,1,float16,fp8,0,2.203013261159261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,4,128,1,float16,float16,0,2.4663146336873374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,4,128,1,float16,fp8,0,2.6178986231486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,8,128,1,float16,float16,0,2.4740907351175943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,16,128,1,float16,fp8,0,1.3808159828186035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,16,128,1,float16,float16,0,2.523594697316488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,16,8,128,1,float16,fp8,0,2.5392533938090005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,1,128,1,float16,float16,0,1.0337279637654622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,1,128,1,float16,fp8,0,1.0781439940134685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,2,128,1,float16,float16,0,1.0705440044403076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,2,128,1,float16,fp8,0,1.0905600388844807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,4,128,1,float16,float16,0,1.2450986703236897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,4,128,1,float16,fp8,0,2.2763306299845376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,8,128,1,float16,float16,0,1.2589066823323567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,1,128,1,float16,float16,0,0.5328319867451986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,16,128,1,float16,float16,0,0.686789353688558
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,16,8,128,1,float16,fp8,0,1.3560266494750977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,4,128,1,float16,float16,0,0.633679986000061
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,16,128,1,float16,fp8,0,0.7335306803385416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,1,128,1,float16,fp8,0,0.5688586632410685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,2,128,1,float16,float16,0,0.5485226710637411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,2,128,1,float16,fp8,0,0.6675733725229899
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,4,128,1,float16,fp8,0,0.6881600220998129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,8,128,1,float16,float16,0,0.6582719882329305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,16,8,128,1,float16,fp8,0,0.7073813279469808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,16,128,1,float16,float16,0,0.3611573378245036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,16,128,1,float16,fp8,0,0.3935146729151408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,1,128,1,float16,float16,0,0.28914133707682294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,1,128,1,float16,fp8,0,0.3055199980735779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,8,128,1,float16,float16,0,0.34544531504313153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,16,128,1,float16,float16,0,0.20654932657877603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,2,128,1,float16,float16,0,0.2983520030975342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,16,128,1,float16,fp8,0,0.22480533520380655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,2,128,1,float16,fp8,0,0.3131626645723979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,4,128,1,float16,float16,0,0.3442986806233724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,4,128,1,float16,fp8,0,0.36745599905649823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,16,8,128,1,float16,fp8,0,0.37915201981862384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,1,128,1,float16,float16,0,0.15643733739852905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,1,128,1,float16,fp8,0,0.16796799500783285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,2,128,1,float16,float16,0,0.16158933440844217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,2,128,1,float16,fp8,0,0.1751413345336914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,4,128,1,float16,float16,0,0.19312000274658203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,4,128,1,float16,fp8,0,0.20697067181269327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,8,128,1,float16,float16,0,0.19840532541275024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,16,8,128,1,float16,fp8,0,0.2142933408419291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,1,128,1,float16,float16,0,2.376250743865967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,1,128,1,float16,fp8,0,2.468560059865316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,2,128,1,float16,fp8,0,2.590165297190348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,4,128,1,float16,float16,0,3.0293334325154624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,4,128,1,float16,fp8,0,3.2280906041463218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,8,128,1,float16,float16,0,3.0958401362101235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,2,128,1,float16,float16,0,6.296762466430664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,16,128,1,float16,float16,0,1.6862400372823079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,1,128,1,float16,fp8,0,1.2743626435597737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,16,128,1,float16,fp8,0,1.685914675394694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,1,128,1,float16,float16,0,1.2048693497975667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,2,128,1,float16,fp8,0,1.3984160423278809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,16,8,128,1,float16,fp8,0,3.1670185724894204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,2,128,1,float16,float16,0,1.2644000053405762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,4,128,1,float16,fp8,0,1.6564319928487141
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,16,128,1,float16,float16,0,0.8409386475880941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,4,128,1,float16,float16,0,3.287482579549154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,8,128,1,float16,float16,0,1.558303991953532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,16,8,128,1,float16,fp8,0,1.638005256652832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,16,128,1,float16,fp8,0,0.8971467018127441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,1,128,1,float16,float16,0,0.6091946760813395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,1,128,1,float16,fp8,0,0.6554826498031616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,2,128,1,float16,float16,0,0.6552693446477255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,2,128,1,float16,fp8,0,0.6848586400349935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,4,128,1,float16,float16,0,1.110869328180949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,4,128,1,float16,fp8,0,0.8453973134358724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,8,128,1,float16,float16,0,0.7888267040252686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,16,128,1,float16,float16,0,0.43506133556365967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,16,8,128,1,float16,fp8,0,0.8520106474558512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,16,128,1,float16,fp8,0,0.4708373149236043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,1,128,1,float16,float16,0,0.32847466071446735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,1,128,1,float16,fp8,0,0.35173332691192627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,2,128,1,float16,float16,0,0.36158398787180585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,2,128,1,float16,fp8,0,0.36154667536417645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,4,128,1,float16,float16,0,0.4061066706975301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,4,128,1,float16,fp8,0,0.44120001792907715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,8,128,1,float16,float16,0,0.4187680085500081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,16,8,128,1,float16,fp8,0,0.4546560049057007
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,16,128,1,float16,float16,0,0.24336532751719156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,4,128,1,float16,float16,0,0.2236479918162028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,16,128,1,float16,fp8,0,0.26258667310078937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,1,128,1,float16,float16,0,0.18217599391937256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,1,128,1,float16,fp8,0,0.19570134083429971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,2,128,1,float16,float16,0,0.1856266657511393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,2,128,1,float16,fp8,0,0.19909866650899252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,4,128,1,float16,fp8,0,0.24307199319203696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,8,128,1,float16,float16,0,0.22978667418162027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,16,8,128,1,float16,fp8,0,0.25171200434366864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,16,128,1,float16,float16,0,0.14131733775138855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,16,128,1,float16,fp8,0,0.15262400110562643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,1,128,1,float16,float16,0,0.09990933537483215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,1,128,1,float16,fp8,0,0.1064906617005666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,2,128,1,float16,float16,0,0.10494400064150493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,2,128,1,float16,fp8,0,0.11583999792734782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,4,128,1,float16,float16,0,0.1243946651617686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,4,128,1,float16,fp8,0,0.1306826670964559
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,8,128,1,float16,float16,0,0.13275200128555298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,16,8,128,1,float16,fp8,0,0.14177067081133524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,1,128,1,float16,float16,0,2.3418025970458984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,1,128,1,float16,fp8,0,2.4781707127889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,2,128,1,float16,float16,0,2.5167039235432944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,2,128,1,float16,fp8,0,2.5703412691752114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,4,128,1,float16,float16,0,3.1644906997680664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,4,128,1,float16,fp8,0,3.4705654780069985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,16,128,1,float16,float16,0,1.7596373558044434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,1,128,1,float16,float16,0,1.1725386778513591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,1,128,1,float16,fp8,0,1.257365306218465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,16,128,1,float16,fp8,0,1.8953226407368977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,8,128,1,float16,fp8,0,3.4607200622558594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,16,8,128,1,float16,float16,0,5.956586837768555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,2,128,1,float16,float16,0,1.2441013654073079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,2,128,1,float16,fp8,0,1.3203946749369304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,4,128,1,float16,float16,0,1.5976319313049316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,16,128,1,float16,float16,0,0.8990346590677897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,4,128,1,float16,fp8,0,1.7546879450480144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,8,128,1,float16,fp8,0,1.761013348897298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,1,128,1,float16,float16,0,0.6027413209279379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,16,128,1,float16,fp8,0,1.1221919854482014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,1,128,1,float16,fp8,0,0.6542239983876547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,2,128,1,float16,float16,0,0.6431839863459269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,2,128,1,float16,fp8,0,0.6733653545379639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,4,128,1,float16,float16,0,0.8212160269419352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,16,8,128,1,float16,float16,0,1.623205343882243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,4,128,1,float16,fp8,0,0.8829120000203451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,8,128,1,float16,float16,0,0.836949348449707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,16,128,1,float16,float16,0,0.4628959894180298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,16,8,128,1,float16,fp8,0,1.0026346842447917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,16,128,1,float16,fp8,0,0.5068693161010742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,4,128,1,float16,fp8,0,0.46649599075317383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,1,128,1,float16,float16,0,0.31480000416437787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,1,128,1,float16,fp8,0,0.3449813524881999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,16,128,1,float16,float16,0,0.24852800369262695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,2,128,1,float16,float16,0,0.34490664800008136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,2,128,1,float16,fp8,0,0.3545813163121541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,4,128,1,float16,float16,0,0.4272799889246623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,8,128,1,float16,float16,0,0.4498453140258789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,16,8,128,1,float16,fp8,0,0.48131732145945233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,16,128,1,float16,fp8,0,0.2692213257153829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,1,128,1,float16,float16,0,0.17534399032592773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,1,128,1,float16,fp8,0,0.19086933135986328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,2,128,1,float16,float16,0,0.187226672967275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,2,128,1,float16,fp8,0,0.19520533084869385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,4,128,1,float16,float16,0,0.23061333100001016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,4,128,1,float16,fp8,0,0.25386667251586914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,8,128,1,float16,float16,0,0.23799999554951987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,16,8,128,1,float16,fp8,0,0.2625280022621155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,16,128,1,float16,float16,0,0.1402346690495809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,16,128,1,float16,fp8,0,0.15499732891718546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,1,128,1,float16,float16,0,0.09631466865539551
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,1,128,1,float16,fp8,0,0.1049066682656606
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,2,128,1,float16,float16,0,0.10051733255386353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,2,128,1,float16,fp8,0,0.10776533683141072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,4,128,1,float16,float16,0,0.12973333398501077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,4,128,1,float16,fp8,0,0.14069333672523499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,8,128,1,float16,float16,0,0.13404800494511923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,16,8,128,1,float16,fp8,0,0.14658133188883463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,16,128,1,float16,float16,0,0.08551466464996338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,16,128,1,float16,fp8,0,0.09089066584904988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,8,128,1,float16,float16,0,0.07412800192832947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,1,128,1,float16,float16,0,0.05585066477457682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,1,128,1,float16,fp8,0,0.06132799883683523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,2,128,1,float16,float16,0,0.06182933350404104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,2,128,1,float16,fp8,0,0.06986666719118755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,4,128,1,float16,float16,0,0.07061333457628886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,4,128,1,float16,fp8,0,0.07774933179219563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,1,128,1,float16,fp8,0,1.592634677886963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,16,8,128,1,float16,fp8,0,0.07977599898974101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,1,128,1,float16,float16,0,1.4520853360493977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,2,128,1,float16,fp8,0,1.6465226809183757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,2,128,1,float16,float16,0,4.531770706176758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,4,128,1,float16,float16,0,2.0843680699666343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,8,128,1,float16,float16,0,2.140549341837565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,16,128,1,float16,float16,0,1.175487995147705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,4,128,1,float16,fp8,0,2.3009546597798667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,16,128,1,float16,fp8,0,1.2902719974517822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,1,128,1,float16,float16,0,0.7339200178782145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,2,128,1,float16,float16,0,0.8076799710591634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,1,128,1,float16,fp8,0,0.8771466414133707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,16,8,128,1,float16,fp8,0,2.343023935953776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,2,128,1,float16,fp8,0,0.8390666643778483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,4,128,1,float16,float16,0,1.048741340637207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,4,128,1,float16,fp8,0,1.15666667620341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,8,128,1,float16,float16,0,1.0677653153737385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,16,8,128,1,float16,fp8,0,1.1794133186340332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,16,128,1,float16,float16,0,0.5993760029474894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,16,128,1,float16,fp8,0,0.6685492992401123
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,1,128,1,float16,float16,0,0.3821706771850586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,1,128,1,float16,fp8,0,0.4182826677958171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,2,128,1,float16,float16,0,0.4283946752548218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,8,128,1,float16,fp8,0,0.6143733263015747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,2,128,1,float16,fp8,0,0.43632535139719647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,4,128,1,float16,float16,0,0.5374346574147543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,4,128,1,float16,fp8,0,0.5973493258158366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,16,8,128,1,float16,float16,0,0.5509706735610962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,16,128,1,float16,float16,0,0.31495465834935504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,16,128,1,float16,fp8,0,0.34405867258707684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,1,128,1,float16,float16,0,0.2077173391977946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,1,128,1,float16,fp8,0,0.22153600056966147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,2,128,1,float16,float16,0,0.22874132792154947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,2,128,1,float16,fp8,0,0.2346400022506714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,4,128,1,float16,float16,0,0.28617600599924725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,16,128,1,float16,fp8,0,0.18862932920455933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,4,128,1,float16,fp8,0,0.3179519971211751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,8,128,1,float16,float16,0,0.2935146689414978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,16,8,128,1,float16,fp8,0,0.327455997467041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,16,128,1,float16,float16,0,0.17253865798314413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,1,128,1,float16,float16,0,0.11597866813341777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,1,128,1,float16,fp8,0,0.12716266512870789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,2,128,1,float16,float16,0,0.12325333555539449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,2,128,1,float16,fp8,0,0.13134933511416116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,4,128,1,float16,float16,0,0.15787733594576517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,4,128,1,float16,fp8,0,0.17562133073806763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,8,128,1,float16,float16,0,0.16179733475049338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,16,8,128,1,float16,fp8,0,0.18019733826319376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,16,128,1,float16,float16,0,0.09839466214179993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,16,128,1,float16,fp8,0,0.11308266719182332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,1,128,1,float16,float16,0,0.06362666686375935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,1,128,1,float16,fp8,0,0.07083733379840851
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,2,128,1,float16,float16,0,0.0690773328145345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,2,128,1,float16,fp8,0,0.07643199960390727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,4,128,1,float16,float16,0,0.08558932940165202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,4,128,1,float16,fp8,0,0.09452266494433086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,8,128,1,float16,float16,0,0.09303466478983562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,16,8,128,1,float16,fp8,0,0.10422933101654053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,16,128,1,float16,float16,0,0.05983466903368632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,16,128,1,float16,fp8,0,0.062218666076660156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,1,128,1,float16,float16,0,0.0432586669921875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,1,128,1,float16,fp8,0,0.04765866696834564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,2,128,1,float16,float16,0,0.04378133515516917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,2,128,1,float16,fp8,0,0.04942933221658071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,4,128,1,float16,float16,0,0.05014933149019877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,4,128,1,float16,fp8,0,0.05682666599750519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,8,128,1,float16,float16,0,0.0498879998922348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,16,8,128,1,float16,fp8,0,0.05894400179386139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,1,128,1,float16,float16,0,1.5417386690775554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,1,128,1,float16,fp8,0,1.681615988413493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,2,128,1,float16,float16,0,1.7154720624287922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,2,128,1,float16,fp8,0,2.993413289388021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,16,128,1,float16,float16,0,1.3460586865743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,4,128,1,float16,float16,0,2.3634026845296225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,16,128,1,float16,fp8,0,1.4347519874572754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,8,128,1,float16,float16,0,2.4344587326049805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,1,128,1,float16,float16,0,0.7740426858266195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,8,128,1,float16,fp8,0,2.622218608856201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,16,4,128,1,float16,fp8,0,2.6612000465393066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,1,128,1,float16,fp8,0,0.9733066558837891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,2,128,1,float16,float16,0,0.8790826797485352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,2,128,1,float16,fp8,0,0.8965173562367758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,4,128,1,float16,float16,0,1.1905866463979085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,4,128,1,float16,fp8,0,1.3411626815795898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,8,128,1,float16,float16,0,1.2002613544464111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,16,8,128,1,float16,fp8,0,1.3464585940043132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,1,128,1,float16,fp8,0,0.44158931573232013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,16,128,1,float16,float16,0,0.6880106925964355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,16,128,1,float16,fp8,0,0.7409813404083252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,1,128,1,float16,float16,0,0.401530663172404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,2,128,1,float16,float16,0,0.44758399327596027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,8,128,1,float16,float16,0,0.6309173504511515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,2,128,1,float16,fp8,0,0.4676533142725627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,4,128,1,float16,float16,0,0.6064053376515707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,16,128,1,float16,fp8,0,0.39714133739471436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,4,128,1,float16,fp8,0,0.6849226951599121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,16,128,1,float16,float16,0,0.35868799686431885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,16,8,128,1,float16,fp8,0,0.7161333560943604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,1,128,1,float16,float16,0,0.212991992632548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,1,128,1,float16,fp8,0,0.23573333024978638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,2,128,1,float16,float16,0,0.23904534180959067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,2,128,1,float16,fp8,0,0.2480000058809916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,4,128,1,float16,float16,0,0.318234662214915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,4,128,1,float16,fp8,0,0.35831467310587567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,1,128,1,float16,fp8,0,0.1317866643269857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,8,128,1,float16,float16,0,0.3318026661872864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,16,8,128,1,float16,fp8,0,0.3742986520131429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,16,128,1,float16,float16,0,0.18920532862345377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,16,128,1,float16,fp8,0,0.21570666631062826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,1,128,1,float16,float16,0,0.11905599633852641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,2,128,1,float16,float16,0,0.13134933511416116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,2,128,1,float16,fp8,0,0.13874133427937826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,4,128,1,float16,float16,0,0.17082667350769043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,4,128,1,float16,fp8,0,0.2083253264427185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,8,128,1,float16,float16,0,0.17892267306645712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,16,128,1,float16,float16,0,0.10634133219718933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,16,8,128,1,float16,fp8,0,0.20037867625554404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,16,128,1,float16,fp8,0,0.12156266967455547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,1,128,1,float16,float16,0,0.06989866495132446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,1,128,1,float16,fp8,0,0.07454399764537811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,2,128,1,float16,float16,0,0.0682773341735204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,2,128,1,float16,fp8,0,0.08256533245245616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,4,128,1,float16,float16,0,0.09437867005666097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,1,128,1,float16,float16,0,0.04068266600370407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,4,128,1,float16,fp8,0,0.10471466183662415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,8,128,1,float16,float16,0,0.09718400239944458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,16,8,128,1,float16,fp8,0,0.11381866534550984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,4,128,1,float16,float16,0,0.04937066634496053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,16,128,1,float16,float16,0,0.0634986658891042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,16,128,1,float16,fp8,0,0.07188266515731812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,1,128,1,float16,fp8,0,0.048485333720842995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,2,128,1,float16,float16,0,0.042597333590189614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,2,128,1,float16,fp8,0,0.04937600096066793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,4,128,1,float16,fp8,0,0.056885331869125366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,8,128,1,float16,float16,0,0.05205333232879639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,1,128,1,float16,fp8,0,0.02898666759332021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,16,8,128,1,float16,fp8,0,0.059989333152770996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,2,128,1,float16,fp8,0,0.03065066784620285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,16,128,1,float16,float16,0,0.03148799886306127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,16,128,1,float16,fp8,0,0.03674133370320002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,1,128,1,float16,float16,0,0.025802666942278545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,2,128,1,float16,float16,0,0.026719999810059864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,4,128,1,float16,float16,0,0.029722665747006733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,4,128,1,float16,fp8,0,0.03475199888149897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,8,128,1,float16,float16,0,0.02991466720898946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,16,8,128,1,float16,fp8,0,0.03480000048875809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,1,128,1,float16,float16,0,1.2825226783752441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,1,128,1,float16,fp8,0,1.3098986943562825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,2,128,1,float16,float16,0,1.3246506849924724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,2,128,1,float16,fp8,0,1.395509401957194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,4,128,1,float16,float16,0,1.9711146354675293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,4,128,1,float16,fp8,0,2.2787572542826333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,16,128,1,float16,float16,0,1.398405392964681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,16,128,1,float16,fp8,0,1.2906773090362549
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,1,128,1,float16,fp8,0,0.6706826686859131
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,1,128,1,float16,float16,0,0.5872000058492025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,2,128,1,float16,float16,0,0.6528693437576294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,8,128,1,float16,float16,0,2.018496036529541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,16,8,128,1,float16,fp8,0,2.3062987327575684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,2,128,1,float16,fp8,0,0.706928014755249
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,4,128,1,float16,float16,0,1.2581813335418701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,4,128,1,float16,fp8,0,1.155728022257487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,16,128,1,float16,float16,0,0.5913920005162557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,8,128,1,float16,float16,0,1.0294346809387207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,16,8,128,1,float16,fp8,0,1.1668372948964436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,1,128,1,float16,fp8,0,0.3462666670481364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,2,128,1,float16,float16,0,0.4252053499221802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,4,128,1,float16,float16,0,0.509989341100057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,2,128,1,float16,fp8,0,0.3675146500269572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,16,128,1,float16,float16,0,0.3304693301518758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,4,128,1,float16,fp8,0,0.5887413422266642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,1,128,1,float16,float16,0,0.16289599736531576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,8,128,1,float16,float16,0,0.5349119901657104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,1,128,1,float16,fp8,0,0.20261865854263306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,1,128,1,float16,float16,0,0.3039146661758423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,16,128,1,float16,fp8,0,0.36535465717315674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,8,128,1,float16,fp8,0,0.6291466554005941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,16,16,128,1,float16,fp8,0,0.6729386647542318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,2,128,1,float16,float16,0,0.18865599234898886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,2,128,1,float16,fp8,0,0.19543466965357462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,4,128,1,float16,float16,0,0.2661386728286743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,4,128,1,float16,fp8,0,0.3240480025609334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,8,128,1,float16,float16,0,0.2778559923171997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,16,8,128,1,float16,fp8,0,0.32289600372314453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,16,128,1,float16,float16,0,0.16250666975975037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,16,128,1,float16,fp8,0,0.19553067286809286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,1,128,1,float16,float16,0,0.09057066837946574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,1,128,1,float16,fp8,0,0.10563199718793233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,2,128,1,float16,float16,0,0.10419199864069621
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,2,128,1,float16,fp8,0,0.12090667088826497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,4,128,1,float16,float16,0,0.14130666851997375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,4,128,1,float16,fp8,0,0.16455466548601785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,8,128,1,float16,float16,0,0.15146133303642273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,16,8,128,1,float16,fp8,0,0.17734400431315103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,16,128,1,float16,float16,0,0.09546132882436116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,16,128,1,float16,fp8,0,0.10999466975529988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,1,128,1,float16,float16,0,0.05226666728655497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,1,128,1,float16,fp8,0,0.05794133245944977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,2,128,1,float16,float16,0,0.054229333996772766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,2,128,1,float16,fp8,0,0.05964800218741099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,4,128,1,float16,float16,0,0.07993599772453308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,4,128,1,float16,fp8,0,0.0888853371143341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,8,128,1,float16,float16,0,0.0824480007092158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,16,8,128,1,float16,fp8,0,0.09921600421269734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,16,128,1,float16,float16,0,0.054805333415667214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,16,128,1,float16,fp8,0,0.06268799801667531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,1,128,1,float16,float16,0,0.03072533259789149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,1,128,1,float16,fp8,0,0.03588266670703888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,8,128,1,float16,fp8,0,0.05018133421738943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,2,128,1,float16,float16,0,0.03311999887228012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,2,128,1,float16,fp8,0,0.0378560001651446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,4,128,1,float16,float16,0,0.038149334490299225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,4,128,1,float16,fp8,0,0.04513066510359446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,16,8,128,1,float16,float16,0,0.041146665811538696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,16,128,1,float16,float16,0,0.02516266703605652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,16,128,1,float16,fp8,0,0.030778666337331135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,1,128,1,float16,float16,0,0.020394666741291683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,1,128,1,float16,fp8,0,0.023258666197458904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,2,128,1,float16,float16,0,0.021274665991465252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,2,128,1,float16,fp8,0,0.024634666740894318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,4,128,1,float16,float16,0,0.023669332265853882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,4,128,1,float16,fp8,0,0.028245332340399425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,8,128,1,float16,float16,0,0.02380799998839696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,16,8,128,1,float16,fp8,0,0.029893333713213604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,16,128,1,float16,float16,0,0.021642667551835377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,16,128,1,float16,fp8,0,0.02658133457104365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,1,128,1,float16,float16,0,0.01903466631968816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,8,128,1,float16,float16,0,0.02080533280968666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,1,128,1,float16,fp8,0,0.02181333303451538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,2,128,1,float16,float16,0,0.019487999379634857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,2,128,1,float16,fp8,0,0.022367998957633972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,4,128,1,float16,float16,0,0.020661332954963047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,4,128,1,float16,fp8,0,0.024682665864626568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,16,8,128,1,float16,fp8,0,0.02643200010061264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,1,128,1,float16,float16,0,0.5025493303934733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,1,128,1,float16,fp8,0,0.5718506574630737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,2,128,1,float16,float16,0,0.5877013206481934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,2,128,1,float16,fp8,0,0.6098613341649374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,4,128,1,float16,float16,0,0.9113066991170248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,8,128,1,float16,fp8,0,1.0899146397908528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,4,128,1,float16,fp8,0,1.0597546895345051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,1,128,1,float16,fp8,0,0.298362672328949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,16,8,128,1,float16,float16,0,0.9304266770680746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,1,128,1,float16,float16,0,0.2595679958661397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,16,128,1,float16,fp8,0,0.5946346521377563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,2,128,1,float16,float16,0,0.30767999092737836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,16,128,1,float16,float16,0,0.5505119959513346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,4,128,1,float16,fp8,0,0.5403146743774414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,16,128,1,float16,fp8,0,0.3242773413658142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,4,128,1,float16,float16,0,0.464687983194987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,8,128,1,float16,float16,0,0.49318933486938477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,8,128,1,float16,fp8,0,0.5828906695048014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,1,128,1,float16,fp8,0,0.15969600280125937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,16,2,128,1,float16,fp8,0,0.321669340133667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,16,128,1,float16,float16,0,0.2844160000483195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,1,128,1,float16,float16,0,0.1418346663316091
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,2,128,1,float16,float16,0,0.1662720044453939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,4,128,1,float16,float16,0,0.24289600054423013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,2,128,1,float16,fp8,0,0.17123200496037802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,4,128,1,float16,fp8,0,0.2827039957046509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,8,128,1,float16,fp8,0,0.2999946673711141
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,16,8,128,1,float16,float16,0,0.2540000081062317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,16,128,1,float16,float16,0,0.15070399641990662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,2,128,1,float16,fp8,0,0.09706133604049683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,16,128,1,float16,fp8,0,0.17561600605646768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,1,128,1,float16,float16,0,0.07825066645940144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,1,128,1,float16,fp8,0,0.09177066882451375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,2,128,1,float16,float16,0,0.08929066856702168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,4,128,1,float16,float16,0,0.13107200463612875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,4,128,1,float16,fp8,0,0.1521013379096985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,8,128,1,float16,float16,0,0.13828266660372415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,16,8,128,1,float16,fp8,0,0.1657386620839437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,16,128,1,float16,float16,0,0.08967467149098714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,16,128,1,float16,fp8,0,0.09977066516876221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,1,128,1,float16,float16,0,0.044026667873064675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,1,128,1,float16,fp8,0,0.051088000337282814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,2,128,1,float16,float16,0,0.04571733375390371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,2,128,1,float16,fp8,0,0.0513919989267985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,4,128,1,float16,float16,0,0.07116800049940745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,4,128,1,float16,fp8,0,0.0830506682395935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,8,128,1,float16,float16,0,0.07663466533025105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,16,8,128,1,float16,fp8,0,0.09156266848246257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,16,128,1,float16,float16,0,0.04757866760094961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,16,128,1,float16,fp8,0,0.05416533350944519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,1,128,1,float16,float16,0,0.025722667574882507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,1,128,1,float16,fp8,0,0.029530666768550873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,8,128,1,float16,fp8,0,0.044346665342648826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,2,128,1,float16,float16,0,0.027322667340437572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,2,128,1,float16,fp8,0,0.03192000091075897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,4,128,1,float16,float16,0,0.03296533226966858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,4,128,1,float16,fp8,0,0.0395359992980957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,16,8,128,1,float16,float16,0,0.03569599986076355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,16,128,1,float16,float16,0,0.022490667800108593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,16,128,1,float16,fp8,0,0.027466667195161183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,1,128,1,float16,float16,0,0.016917333006858826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,1,128,1,float16,fp8,0,0.019765333582957584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,2,128,1,float16,float16,0,0.018405333161354065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,2,128,1,float16,fp8,0,0.021189334491888683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,4,128,1,float16,float16,0,0.020794666061798733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,4,128,1,float16,fp8,0,0.0245919997493426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,8,128,1,float16,float16,0,0.021253332495689392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,16,8,128,1,float16,fp8,0,0.0262773334980011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,2,128,1,float16,fp8,0,0.019146667172511418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,16,128,1,float16,float16,0,0.018464000274737675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,16,128,1,float16,fp8,0,0.023754666248957317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,1,128,1,float16,float16,0,0.016613333175579708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,1,128,1,float16,fp8,0,0.01934933289885521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,2,128,1,float16,float16,0,0.016906666258970898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,16,128,1,float16,fp8,0,0.02141333371400833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,1,128,1,float16,float16,0,0.016154666741689045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,4,128,1,float16,float16,0,0.018138666947682697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,4,128,1,float16,fp8,0,0.020949333906173706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,8,128,1,float16,float16,0,0.018016000588734944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,16,8,128,1,float16,fp8,0,0.022517333428064983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,4,128,1,float16,fp8,0,0.019482667247454327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,16,128,1,float16,float16,0,0.017114666601022083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,1,128,1,float16,fp8,0,0.018432000031073887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,2,128,1,float16,float16,0,0.016122666498025257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,2,128,1,float16,fp8,0,0.018618666877349217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,4,128,1,float16,float16,0,0.01666133354107539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,8,128,1,float16,float16,0,0.016458666572968166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,16,8,128,1,float16,fp8,0,0.020106667031844456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,1,128,1,float16,float16,0,0.25036267439524335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,1,128,1,float16,fp8,0,0.2735840082168579
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,2,128,1,float16,float16,0,0.3006239930788676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,2,128,1,float16,fp8,0,0.2812053362528483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,4,128,1,float16,float16,0,0.4510879913965861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,4,128,1,float16,fp8,0,0.5203359921773275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,16,128,1,float16,float16,0,0.2744479974110921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,16,128,1,float16,fp8,0,0.30001066128412884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,8,128,1,float16,float16,0,0.4753439823786418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,1,128,1,float16,float16,0,0.1343786617120107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,16,8,128,1,float16,fp8,0,0.5329226652781168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,1,128,1,float16,fp8,0,0.1521813372770945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,2,128,1,float16,float16,0,0.15963199734687805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,2,128,1,float16,fp8,0,0.15411733587582907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,4,128,1,float16,float16,0,0.23439999421437582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,16,128,1,float16,float16,0,0.14731199542681375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,16,128,1,float16,fp8,0,0.1596160034338633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,4,128,1,float16,fp8,0,0.2741066614786784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,8,128,1,float16,float16,0,0.24130133787790933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,1,128,1,float16,float16,0,0.07727466523647308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,16,8,128,1,float16,fp8,0,0.27932266394297284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,1,128,1,float16,fp8,0,0.08709866801897685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,2,128,1,float16,float16,0,0.08567999800046285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,2,128,1,float16,fp8,0,0.08776533603668213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,4,128,1,float16,float16,0,0.12713600198427835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,4,128,1,float16,fp8,0,0.14596800009409586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,8,128,1,float16,float16,0,0.13361066579818726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,16,128,1,float16,float16,0,0.08230400085449219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,16,8,128,1,float16,fp8,0,0.15122133493423462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,16,128,1,float16,fp8,0,0.08973866701126099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,1,128,1,float16,float16,0,0.040805332362651825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,1,128,1,float16,fp8,0,0.04640533526738485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,2,128,1,float16,float16,0,0.040735999743143715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,2,128,1,float16,fp8,0,0.04438399771849314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,8,128,1,float16,fp8,0,0.08301333089669545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,4,128,1,float16,float16,0,0.07016000151634216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,4,128,1,float16,fp8,0,0.07618133227030437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,16,8,128,1,float16,float16,0,0.07401066521803538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,16,128,1,float16,float16,0,0.045594667394955955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,16,128,1,float16,fp8,0,0.04836266736189524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,1,128,1,float16,float16,0,0.02312533309062322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,1,128,1,float16,fp8,0,0.02611733227968216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,2,128,1,float16,fp8,0,0.028175999720891316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,4,128,1,float16,float16,0,0.03073599934577942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,2,128,1,float16,float16,0,0.024986666937669117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,4,128,1,float16,fp8,0,0.03551466763019562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,8,128,1,float16,float16,0,0.03301866600910822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,16,8,128,1,float16,fp8,0,0.036570665736993156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,16,128,1,float16,float16,0,0.020736000190178554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,16,128,1,float16,fp8,0,0.02367466688156128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,1,128,1,float16,float16,0,0.015674666812022526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,1,128,1,float16,fp8,0,0.018053332964579265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,2,128,1,float16,float16,0,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,2,128,1,float16,fp8,0,0.01952533299724261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,4,128,1,float16,float16,0,0.019695999721686046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,4,128,1,float16,fp8,0,0.022848000129063923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,8,128,1,float16,float16,0,0.01978133370478948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,16,8,128,1,float16,fp8,0,0.02314666658639908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,16,128,1,float16,float16,0,0.01685333376129468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,16,128,1,float16,fp8,0,0.01945066700379054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,4,128,1,float16,fp8,0,0.019402666638294857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,1,128,1,float16,float16,0,0.014890667051076889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,1,128,1,float16,fp8,0,0.017701332767804463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,2,128,1,float16,float16,0,0.015642666568358738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,2,128,1,float16,fp8,0,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,4,128,1,float16,float16,0,0.016575999557971954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,8,128,1,float16,float16,0,0.016506666938463848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,16,8,128,1,float16,fp8,0,0.01918399954835574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,16,128,1,float16,float16,0,0.01551466683546702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,16,128,1,float16,fp8,0,0.017632000148296356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,1,128,1,float16,float16,0,0.014752000570297241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,1,128,1,float16,fp8,0,0.017210666090250015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,2,128,1,float16,float16,0,0.014570667097965876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,2,128,1,float16,fp8,0,0.016704000532627106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,16,128,1,float16,fp8,0,0.0161013330022494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,4,128,1,float16,float16,0,0.014746667196353277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,4,128,1,float16,fp8,0,0.0169813334941864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,8,128,1,float16,float16,0,0.015509333461523056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,16,8,128,1,float16,fp8,0,0.017866666118303936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,16,128,1,float16,float16,0,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,1,128,1,float16,float16,0,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,1,128,1,float16,fp8,0,0.01575999955336253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,2,128,1,float16,float16,0,0.014581333845853806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,2,128,1,float16,fp8,0,0.015509333461523056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,4,128,1,float16,float16,0,0.013797332843144735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,4,128,1,float16,fp8,0,0.015802666544914246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,8,128,1,float16,float16,0,0.01479999969402949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,16,8,128,1,float16,fp8,0,0.015941333025693893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,1,128,1,float16,float16,0,0.13295466701189676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,1,128,1,float16,fp8,0,0.14877333243687949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,2,128,1,float16,float16,0,0.15714133779207864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,2,128,1,float16,fp8,0,0.15310933192571005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,4,128,1,float16,float16,0,0.23392534255981445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,16,128,1,float16,float16,0,0.15743999679883322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,8,128,1,float16,float16,0,0.24331200122833252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,4,128,1,float16,fp8,0,0.27533332506815594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,16,128,1,float16,fp8,0,0.15340266625086466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,16,8,128,1,float16,fp8,0,0.27987732489903766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,1,128,1,float16,float16,0,0.07758399844169617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,1,128,1,float16,fp8,0,0.08746666709582011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,2,128,1,float16,float16,0,0.08255466818809509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,2,128,1,float16,fp8,0,0.08592533071835835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,4,128,1,float16,float16,0,0.1264906624952952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,1,128,1,float16,float16,0,0.040192000567913055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,4,128,1,float16,fp8,0,0.14639467000961304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,8,128,1,float16,float16,0,0.13462932904561362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,16,128,1,float16,float16,0,0.08685333530108134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,16,8,128,1,float16,fp8,0,0.15226133664449057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,16,128,1,float16,fp8,0,0.08520000179608662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,1,128,1,float16,fp8,0,0.045328001181284584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,2,128,1,float16,float16,0,0.04055999964475632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,2,128,1,float16,fp8,0,0.045125335454940796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,16,128,1,float16,float16,0,0.04795733094215393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,8,128,1,float16,fp8,0,0.08340799808502197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,4,128,1,float16,float16,0,0.0705386648575465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,1,128,1,float16,fp8,0,0.02606933315594991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,2,128,1,float16,float16,0,0.024304000039895374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,4,128,1,float16,fp8,0,0.07578133543332417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,16,8,128,1,float16,float16,0,0.07348800202210744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,16,128,1,float16,fp8,0,0.04320533573627472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,1,128,1,float16,float16,0,0.022837333381175995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,2,128,1,float16,fp8,0,0.028090665737787884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,4,128,1,float16,float16,0,0.0301706666747729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,4,128,1,float16,fp8,0,0.035461333890755974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,8,128,1,float16,float16,0,0.033413333197434746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,16,8,128,1,float16,fp8,0,0.0378560001651446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,16,128,1,float16,float16,0,0.022261333962281544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,16,128,1,float16,fp8,0,0.022853332261244457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,1,128,1,float16,float16,0,0.01586666703224182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,1,128,1,float16,fp8,0,0.01844800015290578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,2,128,1,float16,float16,0,0.016634666671355564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,2,128,1,float16,fp8,0,0.01913600042462349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,4,128,1,float16,float16,0,0.019738666713237762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,4,128,1,float16,fp8,0,0.022917332748572033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,8,128,1,float16,fp8,0,0.023034666975339253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,16,8,128,1,float16,float16,0,0.019365333020687103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,16,128,1,float16,float16,0,0.015098666151364645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,16,128,1,float16,fp8,0,0.015669333438078564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,1,128,1,float16,float16,0,0.014607999473810196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,1,128,1,float16,fp8,0,0.017658667018016178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,2,128,1,float16,float16,0,0.015706667055686314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,2,128,1,float16,fp8,0,0.017077332983414333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,4,128,1,float16,float16,0,0.016773333152135212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,4,128,1,float16,fp8,0,0.019088000059127808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,8,128,1,float16,float16,0,0.016688000410795212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,16,8,128,1,float16,fp8,0,0.019215999792019527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,16,128,1,float16,float16,0,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,16,128,1,float16,fp8,0,0.013770667215188345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,1,128,1,float16,float16,0,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,1,128,1,float16,fp8,0,0.017162666966517765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,8,128,1,float16,fp8,0,0.017263999829689663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,2,128,1,float16,float16,0,0.014794666320085526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,2,128,1,float16,fp8,0,0.016773333152135212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,4,128,1,float16,float16,0,0.015141333142916361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,4,128,1,float16,fp8,0,0.017466666797796886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,16,8,128,1,float16,float16,0,0.014970666418472925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,16,128,1,float16,float16,0,0.013141332815090815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,16,128,1,float16,fp8,0,0.013301332791646322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,1,128,1,float16,float16,0,0.013861333330472311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,1,128,1,float16,fp8,0,0.016122666498025257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,2,128,1,float16,float16,0,0.013594667116800943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,16,128,1,float16,float16,0,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,16,128,1,float16,fp8,0,0.012650666137536367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,2,128,1,float16,fp8,0,0.015578666081031164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,4,128,1,float16,float16,0,0.014576000471909841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,4,128,1,float16,fp8,0,0.015978666643301647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,2,128,1,float16,fp8,0,0.015466666469971338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,8,128,1,float16,float16,0,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,16,8,128,1,float16,fp8,0,0.01700266698996226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,1,128,1,float16,float16,0,0.013631999492645264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,1,128,1,float16,fp8,0,0.015674666812022526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,2,128,1,float16,float16,0,0.013951999445756277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,4,128,1,float16,float16,0,0.013850666582584381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,4,128,1,float16,fp8,0,0.01904533306757609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,8,128,1,float16,float16,0,0.0138026662170887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,16,8,128,1,float16,fp8,0,0.015840000162522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,1,128,1,float16,float16,0,0.07555200159549713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,1,128,1,float16,fp8,0,0.0862666666507721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,2,128,1,float16,float16,0,0.08222400148709615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,2,128,1,float16,fp8,0,0.08643200000127156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,4,128,1,float16,float16,0,0.12446932991345723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,4,128,1,float16,fp8,0,0.14663466811180115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,16,128,1,float16,fp8,0,0.1102293332417806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,8,128,1,float16,float16,0,0.14346667130788168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,16,128,1,float16,float16,0,0.10975466171900432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,16,8,128,1,float16,fp8,0,0.14705600341161093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,1,128,1,float16,float16,0,0.04200533529122671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,1,128,1,float16,fp8,0,0.045279999574025474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,2,128,1,float16,float16,0,0.04050133377313614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,2,128,1,float16,fp8,0,0.04493333399295807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,4,128,1,float16,float16,0,0.07126399874687195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,16,128,1,float16,fp8,0,0.0563679983218511
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,4,128,1,float16,fp8,0,0.07428800066312154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,1,128,1,float16,fp8,0,0.025962665677070618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,8,128,1,float16,float16,0,0.0795413355032603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,16,8,128,1,float16,fp8,0,0.08036266764005025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,16,128,1,float16,float16,0,0.0606826643149058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,1,128,1,float16,float16,0,0.022384000321229298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,2,128,1,float16,float16,0,0.024133334557215374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,2,128,1,float16,fp8,0,0.028597332537174225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,4,128,1,float16,float16,0,0.03030399978160858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,4,128,1,float16,fp8,0,0.035258665680885315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,8,128,1,float16,float16,0,0.03729599962631861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,16,8,128,1,float16,fp8,0,0.035605333745479584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,16,128,1,float16,float16,0,0.02884799987077713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,16,128,1,float16,fp8,0,0.028815999627113342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,1,128,1,float16,float16,0,0.015578666081031164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,1,128,1,float16,fp8,0,0.01838933303952217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,8,128,1,float16,float16,0,0.020784000555674236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,2,128,1,float16,float16,0,0.0169813334941864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,2,128,1,float16,fp8,0,0.01926933353145917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,4,128,1,float16,float16,0,0.01940800001223882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,4,128,1,float16,fp8,0,0.022778667509555817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,16,8,128,1,float16,fp8,0,0.022074667116006214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,16,128,1,float16,float16,0,0.018640000373125076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,16,128,1,float16,fp8,0,0.01903466631968816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,1,128,1,float16,float16,0,0.015029333531856537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,1,128,1,float16,fp8,0,0.017664000391960144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,8,128,1,float16,float16,0,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,8,128,1,float16,fp8,0,0.015482666591803232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,2,128,1,float16,float16,0,0.015311999867359797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,2,128,1,float16,fp8,0,0.017071999609470367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,1,128,1,float16,float16,0,0.014794666320085526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,4,128,1,float16,float16,0,0.01613333324591319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,16,4,128,1,float16,fp8,0,0.019402666638294857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,16,128,1,float16,float16,0,0.013381333400805792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,16,128,1,float16,fp8,0,0.01368533323208491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,1,128,1,float16,fp8,0,0.017157333592573803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,2,128,1,float16,float16,0,0.01451733335852623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,2,128,1,float16,fp8,0,0.016783999900023144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,16,128,1,float16,fp8,0,0.01341333364446958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,4,128,1,float16,float16,0,0.014773332824309668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,4,128,1,float16,fp8,0,0.017514667163292568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,8,128,1,float16,float16,0,0.013365333278973898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,16,8,128,1,float16,fp8,0,0.01357866699496905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,16,128,1,float16,float16,0,0.012938667088747025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,1,128,1,float16,float16,0,0.013850666582584381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,1,128,1,float16,fp8,0,0.015781333049138386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,8,128,1,float16,fp8,0,0.013338666409254074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,2,128,1,float16,float16,0,0.01381333296497663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,2,128,1,float16,fp8,0,0.016672000288963318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,4,128,1,float16,float16,0,0.014010666559139887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,1,128,1,float16,fp8,0,0.015850666910409927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,4,128,1,float16,fp8,0,0.016037333756685257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,16,8,128,1,float16,float16,0,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,4,128,1,float16,float16,0,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,16,128,1,float16,float16,0,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,8,128,1,float16,float16,0,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,16,128,1,float16,fp8,0,0.013066666821638743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,1,128,1,float16,float16,0,0.013647999614477158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,16,128,1,float16,fp8,0,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,2,128,1,float16,float16,0,0.013450667262077332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,2,128,1,float16,fp8,0,0.015568000574906668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,4,128,1,float16,fp8,0,0.015728000551462173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,16,8,128,1,float16,fp8,0,0.012730666746695837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,16,128,1,float16,float16,0,0.011850666254758835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,4,128,1,float16,fp8,0,0.015461333096027374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,1,128,1,float16,float16,0,0.01351999988158544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,1,128,1,float16,fp8,0,0.015599999576807022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,2,128,1,float16,float16,0,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,2,128,1,float16,fp8,0,0.01522133375207583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,4,128,1,float16,float16,0,0.013728000223636627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,8,128,1,float16,float16,0,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,16,8,128,1,float16,fp8,0,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,1,128,1,float16,float16,0,0.04200000067551931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,2,128,1,float16,fp8,0,0.04437866806983948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,1,128,1,float16,fp8,0,0.04520000020662943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,2,128,1,float16,float16,0,0.03988266736268997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,4,128,1,float16,float16,0,0.07547200222810109
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,4,128,1,float16,fp8,0,0.07302399973074596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,16,128,1,float16,float16,0,0.08655466636021932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,8,128,1,float16,float16,0,0.10266666611035664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,16,128,1,float16,fp8,0,0.08275733391443889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,16,8,128,1,float16,fp8,0,0.10429867108662923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,1,128,1,float16,float16,0,0.022826666633288067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,1,128,1,float16,fp8,0,0.025749333202838898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,2,128,1,float16,float16,0,0.02409599969784419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,2,128,1,float16,fp8,0,0.028304000695546467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,16,128,1,float16,float16,0,0.04065066576004028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,4,128,1,float16,float16,0,0.03294933338960012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,4,128,1,float16,fp8,0,0.0349440003434817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,8,128,1,float16,float16,0,0.04967466493447622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,16,8,128,1,float16,fp8,0,0.04710400104522705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,16,128,1,float16,fp8,0,0.0407679999868075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,4,128,1,float16,float16,0,0.02059200033545494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,1,128,1,float16,float16,0,0.01578666642308235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,1,128,1,float16,fp8,0,0.01829333355029424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,8,128,1,float16,fp8,0,0.028181334336598713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,2,128,1,float16,float16,0,0.016821333517630894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,2,128,1,float16,fp8,0,0.019333332777023315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,4,128,1,float16,fp8,0,0.02219199885924657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,16,8,128,1,float16,float16,0,0.0272533322374026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,16,128,1,float16,float16,0,0.024698667228221893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,16,128,1,float16,fp8,0,0.025306666890780132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,1,128,1,float16,float16,0,0.015317333241303762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,1,128,1,float16,fp8,0,0.017765333255132038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,2,128,1,float16,float16,0,0.01482133318980535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,16,128,1,float16,float16,0,0.016757333030303318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,2,128,1,float16,fp8,0,0.017173333714405697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,4,128,1,float16,float16,0,0.014479999740918478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,4,128,1,float16,fp8,0,0.015317333241303762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,2,128,1,float16,float16,0,0.014789332946141561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,8,128,1,float16,float16,0,0.01791999985774358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,16,8,128,1,float16,fp8,0,0.018816000471512478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,16,128,1,float16,fp8,0,0.01695466662446658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,1,128,1,float16,float16,0,0.01469333345691363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,1,128,1,float16,fp8,0,0.01754133279124896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,2,128,1,float16,fp8,0,0.01657066618402799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,4,128,1,float16,float16,0,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,4,128,1,float16,fp8,0,0.013343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,8,128,1,float16,float16,0,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,16,8,128,1,float16,fp8,0,0.013482666263977686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,16,128,1,float16,float16,0,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,2,128,1,float16,fp8,0,0.015466666469971338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,16,128,1,float16,fp8,0,0.01369599997997284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,1,128,1,float16,float16,0,0.013749333719412485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,1,128,1,float16,fp8,0,0.01581866666674614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,2,128,1,float16,float16,0,0.013823999712864557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,4,128,1,float16,float16,0,0.012725333372751871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,16,128,1,float16,fp8,0,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,4,128,1,float16,fp8,0,0.013050666699806849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,8,128,1,float16,float16,0,0.012826666235923767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,2,128,1,float16,fp8,0,0.015418666104475657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,16,8,128,1,float16,fp8,0,0.013295999417702356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,16,128,1,float16,float16,0,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,1,128,1,float16,float16,0,0.0138026662170887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,2,128,1,float16,float16,0,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,1,128,1,float16,fp8,0,0.015504000087579092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,4,128,1,float16,float16,0,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,4,128,1,float16,fp8,0,0.012432000289360682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,8,128,1,float16,float16,0,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,16,8,128,1,float16,fp8,0,0.013269333789745966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,2,128,1,float16,fp8,0,0.014970666418472925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,16,128,1,float16,float16,0,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,16,128,1,float16,fp8,0,0.012960000584522883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,8,128,1,float16,float16,0,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,1,128,1,float16,float16,0,0.013440000514189402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,1,128,1,float16,fp8,0,0.01544533297419548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,2,128,1,float16,float16,0,0.013264000415802002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,4,128,1,float16,float16,0,0.011829332758982977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,1,128,1,float16,fp8,0,0.01524266724785169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,4,128,1,float16,fp8,0,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,16,8,128,1,float16,fp8,0,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,16,128,1,float16,float16,0,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,16,128,1,float16,fp8,0,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,1,128,1,float16,float16,0,0.013450667262077332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,8,128,1,float16,fp8,0,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,2,128,1,float16,float16,0,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,2,128,1,float16,fp8,0,0.014757333944241205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,4,128,1,float16,float16,0,0.011391999820868174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,4,128,1,float16,fp8,0,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,16,8,128,1,float16,float16,0,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,0,0.019909333437681198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,0,0.021669333179791767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,0,0.02640533447265625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,0,0.027376001079877216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,0,0.039162665605545044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,0,0.0422026664018631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,0,0.06530666848023732
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,0,0.06904000043869019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,16,128,1,float16,float16,0,0.06248533229033152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,16,128,1,float16,fp8,0,0.06762666503588359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,0,0.013776000589132309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,0,0.02455466737349828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,0,0.014954666296641031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,0,0.017258666455745697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,0,0.018432000031073887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,16,128,1,float16,fp8,0,0.039647998909155525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,0,0.025983999172846477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,0,0.03654933224121729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,0,0.04022933294375738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,16,128,1,float16,float16,0,0.03714666763941447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,0,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,0,0.013525333255529404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,0,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,0,0.01394133393963178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,0,0.01693333312869072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,0,0.01800000046690305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,0,0.023989332218964893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,0,0.025370667378107708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,0,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,16,128,1,float16,float16,0,0.023189333577950794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,16,128,1,float16,fp8,0,0.024613333245118458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,0,0.01181866725285848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,0,0.012565333396196365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,0,0.01310933381319046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,0,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,0,0.013317332913478216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,0,0.016229332735141117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,0,0.017632000148296356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,16,128,1,float16,float16,0,0.016021333634853363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,16,128,1,float16,fp8,0,0.017407999684413273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,0,0.011685332904259363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,0,0.011978667229413986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,0,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,16,128,1,float16,float16,0,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,0,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,0,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,0,0.01310933381319046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,0,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,0,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,0,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,16,128,1,float16,fp8,0,0.01312000056107839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,0,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,0,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,0,0.011472000430027643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,0,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,0,0.012634667257467905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,0,0.011637333780527115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,0,0.012784000486135483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,16,128,1,float16,float16,0,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,16,128,1,float16,fp8,0,0.013023999830087027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,0,0.010911999891201654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,0,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,0,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,0,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,0,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,0,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,0,0.011642667154471079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,0,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,16,128,1,float16,float16,0,0.012304000556468964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,16,128,1,float16,fp8,0,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,0,0.010890666395425797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,0,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,0,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,0,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,0,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,0,0.012517333030700684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,0,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,0,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,16,128,1,float16,float16,0,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,16,128,1,float16,fp8,0,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,0,0.01099733387430509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,0,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,0,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,0,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,0,0.011349332829316458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,0,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,0,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,0,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,12,128,1,float16,float16,0,6.05894406636556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,1,128,1,float16,float16,0,11.495792388916016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,12,128,1,float16,fp8,0,5.898607889811198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,2,128,1,float16,float16,0,11.568682352701822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,2,128,1,float16,fp8,0,10.578128178914389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,4,128,1,float16,float16,0,11.954325358072916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,4,128,1,float16,fp8,0,11.518768310546875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,1,128,1,float16,float16,0,5.805434544881185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,1,128,1,float16,fp8,0,5.263584136962891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,2,128,1,float16,float16,0,5.46774419148763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,2,128,1,float16,fp8,0,5.405322392781575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,12,128,1,float16,float16,0,3.028421401977539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,12,1,128,1,float16,fp8,0,22.580128987630207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,4,128,1,float16,float16,0,6.245626449584961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,1,128,1,float16,float16,0,2.7256692250569663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,12,4,128,1,float16,fp8,0,6.110144297281901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,12,128,1,float16,fp8,0,3.1346985499064126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,1,128,1,float16,fp8,0,2.973754564921061
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,2,128,1,float16,float16,0,2.7833226521809897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,2,128,1,float16,fp8,0,2.8397385279337564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,12,128,1,float16,float16,0,1.5661226908365886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,12,128,1,float16,fp8,0,1.5697387059529622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,1,128,1,float16,float16,0,1.3836266199747722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,4,128,1,float16,fp8,0,3.126784006754557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,1,128,1,float16,fp8,0,1.4972906112670898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,2,128,1,float16,float16,0,1.443552017211914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,2,128,1,float16,fp8,0,1.4632959365844727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,4,128,1,float16,float16,0,1.5411787033081055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,12,4,128,1,float16,fp8,0,1.5415627161661785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,12,4,128,1,float16,float16,0,7.431477228800456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,1,128,1,float16,float16,0,6.553946812947591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,1,128,1,float16,fp8,0,6.049877166748047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,12,128,1,float16,float16,0,3.503141403198242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,2,128,1,float16,float16,0,6.639589309692383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,2,128,1,float16,fp8,0,6.258752187093099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,4,128,1,float16,float16,0,6.7554931640625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,12,4,128,1,float16,fp8,0,6.856010437011719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,12,128,1,float16,fp8,0,6.898325602213542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,1,128,1,float16,float16,0,3.098400115966797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,1,128,1,float16,fp8,0,3.0488160451253257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,2,128,1,float16,float16,0,3.230629285176595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,2,128,1,float16,fp8,0,3.2818291982014975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,4,128,1,float16,float16,0,3.3903786341349282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,12,128,1,float16,float16,0,1.8039627075195312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,1,128,1,float16,float16,0,1.602901299794515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,12,4,128,1,float16,fp8,0,3.710597356160482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,1,128,1,float16,fp8,0,1.5717919667561848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,12,128,1,float16,fp8,0,3.5761760075887046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,2,128,1,float16,float16,0,1.6339732805887859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,2,128,1,float16,fp8,0,1.6653119723002117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,4,128,1,float16,float16,0,1.7291253407796223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,12,128,1,float16,float16,0,0.9566400051116943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,12,4,128,1,float16,fp8,0,1.7655307451883953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,12,128,1,float16,fp8,0,0.9670506318410238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,1,128,1,float16,float16,0,0.8159893353780111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,1,128,1,float16,fp8,0,1.9476586977640789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,2,128,1,float16,float16,0,0.8574986457824707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,2,128,1,float16,fp8,0,0.8531680107116699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,4,128,1,float16,float16,0,0.899727980295817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,12,4,128,1,float16,fp8,0,0.9534826278686523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,1,128,1,float16,float16,0,4.29307206471761
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,1,128,1,float16,fp8,0,4.252335866292317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,2,128,1,float16,float16,0,4.445210774739583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,12,128,1,float16,float16,0,2.509157339731852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,12,128,1,float16,fp8,0,2.5683679580688477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,4,128,1,float16,float16,0,4.9201812744140625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,1,128,1,float16,float16,0,2.236506621042887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,4,128,1,float16,fp8,0,4.876234690348308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,1,128,1,float16,fp8,0,2.168842633565267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,2,128,1,float16,float16,0,2.277637322743734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,2,128,1,float16,fp8,0,2.24726931254069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,12,2,128,1,float16,fp8,0,8.965365091959635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,4,128,1,float16,float16,0,2.464255968729655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,12,128,1,float16,float16,0,1.3089546362559001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,12,128,1,float16,fp8,0,1.3227626482645671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,1,128,1,float16,float16,0,1.1805386543273926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,12,4,128,1,float16,fp8,0,2.588245391845703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,1,128,1,float16,fp8,0,1.1447892983754475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,2,128,1,float16,fp8,0,1.24673064549764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,12,128,1,float16,float16,0,0.6950613657633463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,4,128,1,float16,float16,0,1.2613120079040527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,2,128,1,float16,float16,0,3.1495841344197593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,12,4,128,1,float16,fp8,0,1.2848426500956218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,2,128,1,float16,float16,0,0.6212533315022787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,12,128,1,float16,fp8,0,0.7163999875386556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,1,128,1,float16,float16,0,0.5854666630427042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,4,128,1,float16,float16,0,0.8630080223083496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,1,128,1,float16,fp8,0,0.6348586479822794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,2,128,1,float16,fp8,0,0.637392004330953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,12,4,128,1,float16,fp8,0,0.6865333716074625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,1,128,1,float16,float16,0,5.998410542805989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,12,128,1,float16,float16,0,3.391301472981771
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,2,128,1,float16,float16,0,6.137978871663411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,12,128,1,float16,fp8,0,3.3522720336914062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,2,128,1,float16,fp8,0,5.766613642374675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,1,128,1,float16,fp8,0,5.66325314839681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,4,128,1,float16,fp8,0,6.536501566569011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,1,128,1,float16,float16,0,2.922405242919922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,1,128,1,float16,fp8,0,2.945199966430664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,2,128,1,float16,float16,0,2.987039883931478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,2,128,1,float16,fp8,0,2.8888266881306968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,12,128,1,float16,float16,0,1.7156426111857097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,12,128,1,float16,fp8,0,1.790885289510091
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,4,128,1,float16,float16,0,3.2228053410847983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,12,4,128,1,float16,fp8,0,3.3058506647745767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,1,128,1,float16,float16,0,1.4239625930786133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,1,128,1,float16,fp8,0,1.545914649963379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,2,128,1,float16,float16,0,1.493402640024821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,2,128,1,float16,fp8,0,1.5156426429748535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,4,128,1,float16,float16,0,1.6499093373616536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,1,128,1,float16,float16,0,0.758570671081543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,1,128,1,float16,fp8,0,0.7661920388539633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,12,4,128,1,float16,float16,0,18.366016387939453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,12,128,1,float16,float16,0,0.8793973128000895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,12,4,128,1,float16,fp8,0,1.7622666358947754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,4,128,1,float16,float16,0,0.8608427047729492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,12,128,1,float16,fp8,0,0.936469316482544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,2,128,1,float16,float16,0,0.7786293029785156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,2,128,1,float16,fp8,0,0.7824319998423258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,1,128,1,float16,fp8,0,0.4253013531366984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,12,4,128,1,float16,fp8,0,0.9016160170237223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,12,128,1,float16,float16,0,0.952245314915975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,12,128,1,float16,fp8,0,0.5081013441085815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,1,128,1,float16,float16,0,0.39821334679921466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,2,128,1,float16,float16,0,0.4240320126215617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,2,128,1,float16,fp8,0,0.43750401337941486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,4,128,1,float16,float16,0,0.4620800018310547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,12,4,128,1,float16,fp8,0,0.4979519844055176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,1,128,1,float16,float16,0,3.213589350382487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,2,128,1,float16,float16,0,3.3729120890299478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,2,128,1,float16,fp8,0,3.4149866104125977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,12,128,1,float16,float16,0,2.069045384724935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,4,128,1,float16,float16,0,3.9032907485961914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,1,128,1,float16,float16,0,1.6598025957743328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,1,128,1,float16,fp8,0,7.053717295328776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,1,128,1,float16,fp8,0,1.7729120254516602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,12,128,1,float16,fp8,0,2.139695962270101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,12,4,128,1,float16,fp8,0,4.0697065989176435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,2,128,1,float16,float16,0,1.726842721303304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,2,128,1,float16,fp8,0,1.7560426394144695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,1,128,1,float16,float16,0,0.8411786556243896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,4,128,1,float16,float16,0,2.022650718688965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,12,128,1,float16,float16,0,1.369050661722819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,12,4,128,1,float16,fp8,0,2.087637265523275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,12,128,1,float16,fp8,0,1.122010628382365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,1,128,1,float16,fp8,0,0.8596426645914713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,2,128,1,float16,float16,0,0.8997973601023356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,2,128,1,float16,fp8,0,0.9116853078206381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,4,128,1,float16,float16,0,1.0089759826660156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,12,128,1,float16,float16,0,0.6238346497217814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,12,4,128,1,float16,fp8,0,1.082250674565633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,12,128,1,float16,fp8,0,0.593946655591329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,1,128,1,float16,float16,0,0.440559983253479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,4,128,1,float16,fp8,0,0.64573868115743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,1,128,1,float16,fp8,0,0.45954668521881104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,2,128,1,float16,float16,0,0.474832018216451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,2,128,1,float16,fp8,0,0.49003732204437256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,2,128,1,float16,float16,0,0.2587626576423645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,12,4,128,1,float16,float16,0,0.5340693394343058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,12,128,1,float16,float16,0,0.31066133578618366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,12,128,1,float16,fp8,0,0.33153067032496136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,1,128,1,float16,float16,0,0.24952000379562378
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,1,128,1,float16,fp8,0,0.2839840054512024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,2,128,1,float16,fp8,0,0.272816002368927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,4,128,1,float16,float16,0,0.2917706569035848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,12,4,128,1,float16,fp8,0,0.3128693302472432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,1,128,1,float16,float16,0,3.059626579284668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,1,128,1,float16,fp8,0,3.1623894373575845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,2,128,1,float16,float16,0,3.342506726582845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,2,128,1,float16,fp8,0,3.3445812861124673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,12,128,1,float16,float16,0,2.0860692660013833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,1,128,1,float16,float16,0,1.5360107421875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,4,128,1,float16,fp8,0,4.104501406351726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,12,128,1,float16,fp8,0,2.351370652516683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,2,128,1,float16,float16,0,1.6452693939208984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,1,128,1,float16,fp8,0,1.7781119346618652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,2,128,1,float16,fp8,0,1.6669012705485027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,12,4,128,1,float16,float16,0,9.596272150675455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,4,128,1,float16,float16,0,1.966767946879069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,12,128,1,float16,float16,0,1.0776960055033367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,12,4,128,1,float16,fp8,0,2.0861600240071616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,12,128,1,float16,fp8,0,1.121440013249715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,1,128,1,float16,float16,0,0.7867733637491862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,1,128,1,float16,fp8,0,0.8278666337331136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,2,128,1,float16,float16,0,1.2364213466644287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,2,128,1,float16,fp8,0,0.8595519860585531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,4,128,1,float16,float16,0,1.0170613129933674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,12,4,128,1,float16,fp8,0,1.0686986446380615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,2,128,1,float16,float16,0,0.5446933507919312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,2,128,1,float16,fp8,0,0.4617653290430705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,12,128,1,float16,float16,0,0.5509973367055258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,1,128,1,float16,float16,0,0.41174932320912677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,1,128,1,float16,fp8,0,0.4384533166885376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,12,128,1,float16,fp8,0,0.5917653242746989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,4,128,1,float16,float16,0,0.5254133145014445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,12,4,128,1,float16,fp8,0,0.5611466566721598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,12,128,1,float16,float16,0,0.39063998063405353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,12,128,1,float16,fp8,0,0.324672003587087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,1,128,1,float16,float16,0,0.22724266846974692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,1,128,1,float16,fp8,0,0.24591465791066489
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,2,128,1,float16,float16,0,0.24690133333206177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,2,128,1,float16,fp8,0,0.25944532950719196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,1,128,1,float16,fp8,0,0.13582400480906168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,2,128,1,float16,float16,0,0.1397599975268046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,4,128,1,float16,float16,0,0.2882879972457886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,12,4,128,1,float16,fp8,0,0.3049386739730835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,4,128,1,float16,fp8,0,0.1680319905281067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,12,128,1,float16,float16,0,0.1801919937133789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,12,128,1,float16,fp8,0,0.18739734093348184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,1,128,1,float16,float16,0,0.12451733152071635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,2,128,1,float16,fp8,0,0.14639467000961304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,12,4,128,1,float16,float16,0,0.16337600350379944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,1,128,1,float16,float16,0,1.827605406443278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,1,128,1,float16,fp8,0,1.8896640141805012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,2,128,1,float16,float16,0,1.9706400235493977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,2,128,1,float16,fp8,0,3.1940587361653647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,4,128,1,float16,float16,0,2.455498695373535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,12,128,1,float16,fp8,0,1.4234293301900227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,12,128,1,float16,float16,0,1.3387840588887532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,1,128,1,float16,float16,0,0.9417493343353271
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,12,4,128,1,float16,fp8,0,2.6640267372131348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,1,128,1,float16,fp8,0,0.9682133197784424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,2,128,1,float16,float16,0,1.0247893333435059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,2,128,1,float16,fp8,0,1.490085283915202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,4,128,1,float16,float16,0,1.2467520236968994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,12,128,1,float16,float16,0,0.6860000292460123
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,12,4,128,1,float16,fp8,0,1.365338643391927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,12,128,1,float16,fp8,0,0.7427413463592529
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,1,128,1,float16,float16,0,0.4723840157190959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,1,128,1,float16,fp8,0,0.5051680008570353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,2,128,1,float16,float16,0,0.5406879981358846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,2,128,1,float16,fp8,0,0.7007786432902018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,1,128,1,float16,fp8,0,0.27588800589243573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,4,128,1,float16,float16,0,0.6378613313039144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,12,128,1,float16,float16,0,0.360586682955424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,12,4,128,1,float16,fp8,0,0.7053919633229574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,12,128,1,float16,fp8,0,0.3936906655629476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,1,128,1,float16,float16,0,0.2561013301213582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,2,128,1,float16,float16,0,0.28810666004816693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,2,128,1,float16,fp8,0,0.3181440035502116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,4,128,1,float16,float16,0,0.34036799271901447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,2,128,1,float16,float16,0,0.16381333271662393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,12,4,128,1,float16,fp8,0,0.37724268436431885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,12,128,1,float16,float16,0,0.20454933245976767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,12,128,1,float16,fp8,0,0.22086399793624878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,1,128,1,float16,float16,0,0.1456160048643748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,1,128,1,float16,fp8,0,0.15587199727694193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,2,128,1,float16,fp8,0,0.1694506605466207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,4,128,1,float16,float16,0,0.18861865997314453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,2,128,1,float16,float16,0,0.08964799841245015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,12,4,128,1,float16,fp8,0,0.20994667212168375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,12,128,1,float16,float16,0,0.11972266435623169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,12,128,1,float16,fp8,0,0.13296533624331155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,1,128,1,float16,float16,0,0.08435199658075969
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,1,128,1,float16,fp8,0,0.09470400214195251
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,2,128,1,float16,fp8,0,0.09841600060462952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,4,128,1,float16,float16,0,0.1016480028629303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,12,4,128,1,float16,fp8,0,0.11311466495196025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,1,128,1,float16,float16,0,1.7898027102152507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,1,128,1,float16,fp8,0,1.928042729695638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,4,128,1,float16,float16,0,2.6272640228271484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,2,128,1,float16,float16,0,2.0601654052734375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,12,128,1,float16,fp8,0,1.5510506629943848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,1,128,1,float16,float16,0,0.9092106819152832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,2,128,1,float16,fp8,0,2.068101406097412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,12,128,1,float16,float16,0,1.4571199417114258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,12,4,128,1,float16,fp8,0,3.597349484761556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,1,128,1,float16,fp8,0,0.9698932965596517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,2,128,1,float16,float16,0,1.0508320331573486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,2,128,1,float16,fp8,0,1.0747946898142497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,4,128,1,float16,float16,0,1.3307466506958008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,12,128,1,float16,float16,0,0.7426026662190756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,12,4,128,1,float16,fp8,0,1.4756266276041667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,12,128,1,float16,fp8,0,0.8294400374094645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,1,128,1,float16,float16,0,0.4668533404668172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,1,128,1,float16,fp8,0,0.5076533158620199
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,2,128,1,float16,float16,0,0.5505280097325643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,2,128,1,float16,fp8,0,0.549509326616923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,12,128,1,float16,fp8,0,0.41610666116078693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,4,128,1,float16,float16,0,0.6867520014444987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,12,4,128,1,float16,fp8,0,0.7594079971313477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,12,128,1,float16,float16,0,0.4052853186925252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,1,128,1,float16,float16,0,0.25376532475153607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,1,128,1,float16,fp8,0,0.272053341070811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,2,128,1,float16,float16,0,0.288975993792216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,2,128,1,float16,fp8,0,0.2940266728401184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,4,128,1,float16,float16,0,0.3622986475626628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,12,4,128,1,float16,fp8,0,0.40035732587178546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,12,128,1,float16,float16,0,0.21557867527008057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,12,128,1,float16,fp8,0,0.22952532768249512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,1,128,1,float16,float16,0,0.14061333735783896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,1,128,1,float16,fp8,0,0.15371732910474142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,2,128,1,float16,float16,0,0.15902933478355408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,2,128,1,float16,fp8,0,0.16564266880353293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,4,128,1,float16,float16,0,0.19733333587646484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,12,4,128,1,float16,fp8,0,0.2180746595064799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,12,128,1,float16,float16,0,0.11813333630561829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,12,128,1,float16,fp8,0,0.13221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,1,128,1,float16,float16,0,0.0777759999036789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,1,128,1,float16,fp8,0,0.08588799834251404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,2,128,1,float16,float16,0,0.08833600083986919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,2,128,1,float16,fp8,0,0.09204266468683879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,4,128,1,float16,float16,0,0.11016533772150676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,12,4,128,1,float16,fp8,0,0.11588266491889954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,12,128,1,float16,float16,0,0.06981866558392842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,12,128,1,float16,fp8,0,0.07400533556938171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,1,128,1,float16,float16,0,0.0507893313964208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,1,128,1,float16,fp8,0,0.05587733288606008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,2,128,1,float16,float16,0,0.052842666705449425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,2,128,1,float16,fp8,0,0.059546664357185364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,4,128,1,float16,float16,0,0.06159466505050659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,12,4,128,1,float16,fp8,0,0.06995733578999837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,1,128,1,float16,fp8,0,1.2097386519114177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,1,128,1,float16,float16,0,1.1139787038167317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,2,128,1,float16,float16,0,1.276261329650879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,2,128,1,float16,fp8,0,1.3359252611796062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,4,128,1,float16,float16,0,1.7361547152201335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,12,128,1,float16,float16,0,0.9789120356241862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,12,128,1,float16,fp8,0,1.062538703282674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,1,128,1,float16,float16,0,0.7367839813232422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,12,4,128,1,float16,fp8,0,1.9583199818929036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,1,128,1,float16,fp8,0,0.6222826639811198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,2,128,1,float16,float16,0,0.67412797609965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,2,128,1,float16,fp8,0,0.6862293084462484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,4,128,1,float16,float16,0,0.8803893725077311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,1,128,1,float16,fp8,0,0.3256160020828247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,2,128,1,float16,float16,0,0.35981865723927814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,2,128,1,float16,fp8,0,0.36185065905253094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,12,4,128,1,float16,fp8,0,1.0078612963358562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,12,128,1,float16,float16,0,0.5077866713205973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,12,128,1,float16,fp8,0,0.5882293383280436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,1,128,1,float16,float16,0,0.30190932750701904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,4,128,1,float16,float16,0,0.46164798736572266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,12,4,128,1,float16,fp8,0,0.5161653359731039
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,12,128,1,float16,float16,0,0.2681066592534383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,12,128,1,float16,fp8,0,0.2956586678822835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,1,128,1,float16,float16,0,0.1672053337097168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,1,128,1,float16,fp8,0,0.18228799104690552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,2,128,1,float16,float16,0,0.19620267550150552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,2,128,1,float16,fp8,0,0.19818133115768433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,1,128,1,float16,fp8,0,0.10221866766611735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,4,128,1,float16,float16,0,0.24524799982706705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,12,4,128,1,float16,fp8,0,0.2765653332074483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,12,128,1,float16,float16,0,0.14774399995803833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,12,128,1,float16,fp8,0,0.16334399580955505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,1,128,1,float16,float16,0,0.09523199995358785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,2,128,1,float16,float16,0,0.10899733503659566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,2,128,1,float16,fp8,0,0.11155200004577637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,4,128,1,float16,float16,0,0.13587733109792074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,12,4,128,1,float16,fp8,0,0.15479466319084167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,12,128,1,float16,float16,0,0.08489066362380981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,12,128,1,float16,fp8,0,0.09670399626096089
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,1,128,1,float16,float16,0,0.053770666321118675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,1,128,1,float16,fp8,0,0.06160533428192139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,2,128,1,float16,float16,0,0.05937600135803223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,2,128,1,float16,fp8,0,0.06525333225727081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,4,128,1,float16,float16,0,0.06900266806284587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,12,4,128,1,float16,fp8,0,0.08310933411121368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,12,128,1,float16,float16,0,0.050197333097457886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,12,128,1,float16,fp8,0,0.05527999997138977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,1,128,1,float16,float16,0,0.03793599953254064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,1,128,1,float16,fp8,0,0.0436160018046697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,2,128,1,float16,float16,0,0.03990933299064636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,2,128,1,float16,fp8,0,0.04561600089073181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,4,128,1,float16,float16,0,0.04348800083001455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,12,4,128,1,float16,fp8,0,0.05087999999523163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,1,128,1,float16,float16,0,1.3660906155904133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,1,128,1,float16,fp8,0,1.4494345982869465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,2,128,1,float16,float16,0,1.6392213503519695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,2,128,1,float16,fp8,0,1.613418738047282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,4,128,1,float16,float16,0,2.819125175476074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,12,128,1,float16,float16,0,1.2302827040354412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,1,128,1,float16,float16,0,0.7038453420003256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,12,128,1,float16,fp8,0,1.344912052154541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,12,4,128,1,float16,fp8,0,2.458298683166504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,1,128,1,float16,fp8,0,0.7347093423207601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,2,128,1,float16,float16,0,0.8134187062581381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,12,128,1,float16,float16,0,0.6255306800206503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,1,128,1,float16,float16,0,0.3649866580963135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,2,128,1,float16,fp8,0,0.8717119693756104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,4,128,1,float16,float16,0,1.1209332942962646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,12,4,128,1,float16,fp8,0,1.2521119912465413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,1,128,1,float16,fp8,0,0.38097067674001056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,2,128,1,float16,float16,0,0.4307519992192586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,12,128,1,float16,fp8,0,0.6850612958272299
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,2,128,1,float16,fp8,0,0.4272693395614624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,4,128,1,float16,float16,0,0.5825706720352173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,12,4,128,1,float16,fp8,0,0.6435253222783407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,12,128,1,float16,float16,0,0.3272426724433899
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,12,128,1,float16,fp8,0,0.36004801591237384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,1,128,1,float16,float16,0,0.19289066394170126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,1,128,1,float16,fp8,0,0.20429333051045737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,2,128,1,float16,float16,0,0.23030932744344076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,2,128,1,float16,fp8,0,0.2293013334274292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,1,128,1,float16,float16,0,0.10734400153160095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,1,128,1,float16,fp8,0,0.11622933546702068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,4,128,1,float16,float16,0,0.30110400915145874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,12,128,1,float16,float16,0,0.17478932936986288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,12,4,128,1,float16,fp8,0,0.336298664410909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,12,128,1,float16,fp8,0,0.19642666975657144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,2,128,1,float16,float16,0,0.1250986655553182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,2,128,1,float16,fp8,0,0.127920001745224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,4,128,1,float16,float16,0,0.15972266594568887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,12,4,128,1,float16,fp8,0,0.18011200428009033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,12,128,1,float16,float16,0,0.09735467036565144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,4,128,1,float16,float16,0,0.08797867099444072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,12,128,1,float16,fp8,0,0.10972799857457478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,1,128,1,float16,float16,0,0.05936000247796377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,2,128,1,float16,float16,0,0.06606400012969971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,1,128,1,float16,fp8,0,0.06347733239332835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,2,128,1,float16,fp8,0,0.06735999882221222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,12,4,128,1,float16,fp8,0,0.09453866879145305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,2,128,1,float16,fp8,0,0.04491733511288961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,12,128,1,float16,float16,0,0.055386667450269066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,12,128,1,float16,fp8,0,0.057616000374158226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,1,128,1,float16,float16,0,0.03746666759252548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,1,128,1,float16,fp8,0,0.04045866678158442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,1,128,1,float16,float16,0,0.02571733295917511
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,1,128,1,float16,fp8,0,0.028186666468779247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,2,128,1,float16,float16,0,0.04106133431196213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,4,128,1,float16,float16,0,0.046336000164349876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,12,4,128,1,float16,fp8,0,0.05353599786758423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,4,128,1,float16,fp8,0,0.034389334420363106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,12,128,1,float16,float16,0,0.030960001051425934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,12,128,1,float16,fp8,0,0.03514666606982549
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,2,128,1,float16,float16,0,0.027647999425729115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,2,128,1,float16,fp8,0,0.030016000072161358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,12,4,128,1,float16,float16,0,0.029909332593282063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,1,128,1,float16,float16,0,0.9065120220184326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,1,128,1,float16,fp8,0,1.026037295659383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,2,128,1,float16,float16,0,1.1727360089619954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,2,128,1,float16,fp8,0,1.2083946863810222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,12,128,1,float16,float16,0,0.9973333676656088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,12,128,1,float16,fp8,0,1.1386880079905193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,1,128,1,float16,float16,0,0.4647786617279053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,4,128,1,float16,float16,0,1.7325545946757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,1,128,1,float16,fp8,0,0.5259519815444946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,12,4,128,1,float16,fp8,0,2.0679893493652344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,12,128,1,float16,float16,0,0.5097759962081909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,2,128,1,float16,float16,0,0.5968853235244751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,2,128,1,float16,fp8,0,0.6189493338267008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,1,128,1,float16,float16,0,0.24503467480341592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,4,128,1,float16,float16,0,0.893999973932902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,12,128,1,float16,fp8,0,0.6264426708221436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,12,4,128,1,float16,fp8,0,1.0513813495635986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,2,128,1,float16,float16,0,0.3227840065956116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,2,128,1,float16,fp8,0,0.32152533531188965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,4,128,1,float16,float16,0,0.45397865772247314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,1,128,1,float16,fp8,0,0.2763413389523824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,1,128,1,float16,fp8,0,0.14967999855677286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,12,128,1,float16,fp8,0,0.3223466674486796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,12,128,1,float16,float16,0,0.26442132393519086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,12,4,128,1,float16,fp8,0,0.5398133198420206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,1,128,1,float16,float16,0,0.13206400473912558
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,2,128,1,float16,float16,0,0.16638933618863425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,2,128,1,float16,fp8,0,0.17569067080815634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,4,128,1,float16,float16,0,0.2351413369178772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,12,4,128,1,float16,fp8,0,0.2831679979960124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,12,128,1,float16,float16,0,0.14314132928848267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,12,128,1,float16,fp8,0,0.1739306648572286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,1,128,1,float16,float16,0,0.07494933406511943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,1,128,1,float16,fp8,0,0.08721066514650981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,2,128,1,float16,float16,0,0.09314133723576863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,2,128,1,float16,fp8,0,0.1018293301264445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,4,128,1,float16,float16,0,0.12520533800125122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,2,128,1,float16,float16,0,0.047007997830708824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,12,4,128,1,float16,fp8,0,0.15291200081507364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,12,128,1,float16,float16,0,0.08014933268229167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,12,128,1,float16,fp8,0,0.09640533725420634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,1,128,1,float16,float16,0,0.04128533353408178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,1,128,1,float16,fp8,0,0.0489279975493749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,2,128,1,float16,fp8,0,0.05409066875775655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,4,128,1,float16,float16,0,0.07072533170382182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,12,4,128,1,float16,fp8,0,0.07543466488520305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,12,128,1,float16,float16,0,0.04468800127506256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,12,128,1,float16,fp8,0,0.048986668388048805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,1,128,1,float16,float16,0,0.027509334186712902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,1,128,1,float16,fp8,0,0.033413333197434746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,12,128,1,float16,float16,0,0.024058667321999867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,2,128,1,float16,float16,0,0.02977066735426585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,2,128,1,float16,fp8,0,0.03726933399836222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,4,128,1,float16,float16,0,0.035114665826161705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,2,128,1,float16,float16,0,0.02083733429511388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,12,4,128,1,float16,fp8,0,0.0443146675825119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,12,128,1,float16,fp8,0,0.030154667794704437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,4,128,1,float16,fp8,0,0.02972800036271413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,1,128,1,float16,float16,0,0.0207893339296182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,1,128,1,float16,fp8,0,0.023445333043734234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,2,128,1,float16,fp8,0,0.025674665967623394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,1,128,1,float16,fp8,0,0.02231466770172119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,12,4,128,1,float16,float16,0,0.023402666052182514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,12,128,1,float16,float16,0,0.021301334102948506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,12,128,1,float16,fp8,0,0.0262719988822937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,1,128,1,float16,float16,0,0.018794666975736618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,2,128,1,float16,float16,0,0.019391999890406925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,2,128,1,float16,fp8,0,0.024266667664051056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,4,128,1,float16,float16,0,0.020479999482631683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,12,4,128,1,float16,fp8,0,0.026165333886941273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,1,128,1,float16,float16,0,0.3967573245366414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,1,128,1,float16,fp8,0,0.4524799982706706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,1,128,1,float16,fp8,0,0.2382026712099711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,1,128,1,float16,float16,0,0.20907199382781982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,2,128,1,float16,float16,0,0.5396053393681844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,2,128,1,float16,fp8,0,0.5452106793721517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,2,128,1,float16,float16,0,0.2707093358039856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,2,128,1,float16,fp8,0,0.2884053389231364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,4,128,1,float16,fp8,0,0.9749546845753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,12,4,128,1,float16,float16,0,0.8162666956583658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,4,128,1,float16,float16,0,0.41498132546742755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,4,128,1,float16,fp8,0,0.4976160128911336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,12,128,1,float16,float16,0,0.24804266293843588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,12,128,1,float16,float16,0,0.4776000181833903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,12,12,128,1,float16,fp8,0,0.5284800132115682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,12,128,1,float16,fp8,0,0.28386133909225464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,1,128,1,float16,float16,0,0.11499733726183574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,2,128,1,float16,float16,0,0.1506613294283549
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,1,128,1,float16,fp8,0,0.13081600268681845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,2,128,1,float16,fp8,0,0.15661866466204324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,4,128,1,float16,float16,0,0.21681066354115805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,1,128,1,float16,float16,0,0.06679466863473256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,1,128,1,float16,fp8,0,0.07650133470694225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,12,128,1,float16,fp8,0,0.15523200233777365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,2,128,1,float16,float16,0,0.08436800042788188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,2,128,1,float16,fp8,0,0.09089600046475728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,4,128,1,float16,fp8,0,0.1415733297665914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,4,128,1,float16,float16,0,0.11757333079973857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,12,128,1,float16,float16,0,0.07400533556938171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,12,128,1,float16,fp8,0,0.08743466933568318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,1,128,1,float16,float16,0,0.03572800010442734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,12,12,128,1,float16,float16,0,0.13412266969680786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,1,128,1,float16,fp8,0,0.040762667854626976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,12,128,1,float16,float16,0,0.040234667559464775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,12,4,128,1,float16,fp8,0,0.2630506753921509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,12,128,1,float16,fp8,0,0.0476746658484141
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,1,128,1,float16,float16,0,0.023120000958442688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,2,128,1,float16,float16,0,0.040896000961462654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,2,128,1,float16,float16,0,0.025600001215934753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,2,128,1,float16,fp8,0,0.04701333244641622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,4,128,1,float16,float16,0,0.06576000154018402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,12,4,128,1,float16,fp8,0,0.0697813332080841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,1,128,1,float16,fp8,0,0.02756800005833308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,2,128,1,float16,fp8,0,0.03173866619666418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,4,128,1,float16,float16,0,0.030805334448814392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,12,4,128,1,float16,fp8,0,0.038693333665529885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,12,128,1,float16,float16,0,0.021397332350413006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,4,128,1,float16,float16,0,0.020741333564122517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,12,128,1,float16,fp8,0,0.02741866558790207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,1,128,1,float16,float16,0,0.01666133354107539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,1,128,1,float16,fp8,0,0.01947733387351036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,2,128,1,float16,float16,0,0.018042666216691334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,2,128,1,float16,fp8,0,0.022416000564893086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,12,4,128,1,float16,fp8,0,0.025797332326571148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,2,128,1,float16,fp8,0,0.02042666698495547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,12,128,1,float16,float16,0,0.018506667266289394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,12,128,1,float16,fp8,0,0.023413332800070446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,1,128,1,float16,float16,0,0.016528000434239704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,1,128,1,float16,fp8,0,0.01874133323629697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,2,128,1,float16,float16,0,0.016271999726692837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,4,128,1,float16,float16,0,0.018191999445358913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,12,4,128,1,float16,fp8,0,0.021984001000722248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,12,128,1,float16,float16,0,0.01657066618402799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,12,128,1,float16,fp8,0,0.0215786670645078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,1,128,1,float16,float16,0,0.01607999950647354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,1,128,1,float16,fp8,0,0.01858666663368543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,2,128,1,float16,float16,0,0.016063999384641647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,2,128,1,float16,fp8,0,0.020069333414236706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,4,128,1,float16,float16,0,0.016719999412695568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,12,4,128,1,float16,fp8,0,0.02027733375628789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,1,128,1,float16,float16,0,0.19962133963902792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,1,128,1,float16,fp8,0,0.21854400634765625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,2,128,1,float16,float16,0,0.2711413304011027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,2,128,1,float16,fp8,0,0.2629706660906474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,12,128,1,float16,fp8,0,0.2609333395957947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,12,128,1,float16,float16,0,0.24100800355275473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,1,128,1,float16,float16,0,0.10922132929166158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,1,128,1,float16,fp8,0,0.12416533629099528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,4,128,1,float16,float16,0,0.401253342628479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,2,128,1,float16,float16,0,0.14358400305112204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,12,4,128,1,float16,fp8,0,0.4750773509343465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,2,128,1,float16,fp8,0,0.14388799667358398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,4,128,1,float16,fp8,0,0.24740799268086752
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,12,4,128,1,float16,float16,0,0.2107093334197998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,12,128,1,float16,float16,0,0.130021333694458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,12,128,1,float16,fp8,0,0.14243732889493307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,1,128,1,float16,float16,0,0.06320000191529591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,1,128,1,float16,fp8,0,0.0703893353541692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,2,128,1,float16,float16,0,0.08230400085449219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,2,128,1,float16,fp8,0,0.08286933104197185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,4,128,1,float16,float16,0,0.11408000191052754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,12,4,128,1,float16,fp8,0,0.13665599624315897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,1,128,1,float16,fp8,0,0.03535466641187668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,12,128,1,float16,float16,0,0.07201600074768066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,12,128,1,float16,fp8,0,0.07875733574231465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,4,128,1,float16,fp8,0,0.05955199897289276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,1,128,1,float16,float16,0,0.03146133323510488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,12,128,1,float16,fp8,0,0.04314666489760081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,2,128,1,float16,fp8,0,0.039808000127474465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,2,128,1,float16,float16,0,0.037589333951473236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,12,4,128,1,float16,float16,0,0.061759998401006065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,12,128,1,float16,float16,0,0.037151999771595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,1,128,1,float16,float16,0,0.021568000316619873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,1,128,1,float16,fp8,0,0.02565866708755493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,2,128,1,float16,float16,0,0.0249439999461174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,2,128,1,float16,fp8,0,0.028757333755493164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,4,128,1,float16,float16,0,0.029711998999118805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,12,4,128,1,float16,fp8,0,0.03514666606982549
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,2,128,1,float16,fp8,0,0.019413333386182785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,4,128,1,float16,float16,0,0.019253333409627277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,12,128,1,float16,float16,0,0.02019199977318446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,4,128,1,float16,fp8,0,0.02276800076166789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,12,128,1,float16,fp8,0,0.023071999351183575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,1,128,1,float16,float16,0,0.015024000157912573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,1,128,1,float16,float16,0,0.01534933348496755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,1,128,1,float16,fp8,0,0.017743999759356182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,12,2,128,1,float16,float16,0,0.016757333030303318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,12,128,1,float16,float16,0,0.016682667036851246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,4,128,1,float16,fp8,0,0.01923199991385142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,12,128,1,float16,fp8,0,0.019445333629846573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,12,128,1,float16,fp8,0,0.017717332889636356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,1,128,1,float16,fp8,0,0.017498667041460674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,2,128,1,float16,float16,0,0.015263999501864115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,2,128,1,float16,float16,0,0.014469332993030548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,2,128,1,float16,fp8,0,0.017477333545684814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,12,4,128,1,float16,float16,0,0.016794666647911072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,12,128,1,float16,float16,0,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,1,128,1,float16,float16,0,0.014560000350077948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,1,128,1,float16,fp8,0,0.01729600007335345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,2,128,1,float16,fp8,0,0.01720533271630605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,4,128,1,float16,float16,0,0.015082667271296183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,12,4,128,1,float16,fp8,0,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,12,128,1,float16,float16,0,0.014277332772811254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,12,128,1,float16,fp8,0,0.015781333049138386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,1,128,1,float16,float16,0,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,1,128,1,float16,fp8,0,0.015589332828919092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,2,128,1,float16,float16,0,0.013807999591032663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,2,128,1,float16,fp8,0,0.01599466676513354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,4,128,1,float16,float16,0,0.014021333307027817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,12,4,128,1,float16,fp8,0,0.016122666498025257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,1,128,1,float16,float16,0,0.10851732889811198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,1,128,1,float16,fp8,0,0.12306666374206543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,2,128,1,float16,float16,0,0.14181333780288696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,2,128,1,float16,fp8,0,0.14642666776974997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,4,128,1,float16,float16,0,0.2281173268953959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,12,128,1,float16,float16,0,0.13711466391881308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,2,128,1,float16,float16,0,0.08210133512814839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,1,128,1,float16,float16,0,0.06379733482996623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,12,4,128,1,float16,fp8,0,0.26769065856933594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,12,128,1,float16,fp8,0,0.13873066504796347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,4,128,1,float16,float16,0,0.12285866340001424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,12,128,1,float16,float16,0,0.07549866537253062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,1,128,1,float16,fp8,0,0.07070399820804596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,2,128,1,float16,fp8,0,0.08258133133252461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,12,4,128,1,float16,fp8,0,0.14568000038464865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,12,128,1,float16,fp8,0,0.07584000130494435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,1,128,1,float16,float16,0,0.031002665559450786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,1,128,1,float16,fp8,0,0.03526400029659271
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,2,128,1,float16,float16,0,0.03880000114440918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,2,128,1,float16,fp8,0,0.04111466556787491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,4,128,1,float16,float16,0,0.06529599924882253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,12,4,128,1,float16,fp8,0,0.06261866788069408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,12,128,1,float16,float16,0,0.036501333117485046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,2,128,1,float16,fp8,0,0.028170667588710785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,12,128,1,float16,fp8,0,0.03307733436425527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,1,128,1,float16,float16,0,0.022266666094462078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,1,128,1,float16,fp8,0,0.025637333591779072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,2,128,1,float16,float16,0,0.024160000185171764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,4,128,1,float16,float16,0,0.02942399928967158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,12,4,128,1,float16,fp8,0,0.03482666611671448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,2,128,1,float16,float16,0,0.01666133354107539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,12,128,1,float16,float16,0,0.02110933264096578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,2,128,1,float16,fp8,0,0.019274666905403137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,12,128,1,float16,fp8,0,0.022085333863894146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,1,128,1,float16,float16,0,0.015418666104475657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,1,128,1,float16,fp8,0,0.018058666338523228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,1,128,1,float16,float16,0,0.014869333555301031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,4,128,1,float16,float16,0,0.018965333700180054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,12,4,128,1,float16,fp8,0,0.022389332453409832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,12,128,1,float16,float16,0,0.014752000570297241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,12,128,1,float16,fp8,0,0.015626666446526844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,4,128,1,float16,fp8,0,0.019050666441520054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,1,128,1,float16,fp8,0,0.017279999951521557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,12,128,1,float16,float16,0,0.013125333935022354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,2,128,1,float16,float16,0,0.015509333461523056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,2,128,1,float16,fp8,0,0.017338667064905167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,12,4,128,1,float16,float16,0,0.01609066625436147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,12,128,1,float16,fp8,0,0.013967999567588171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,1,128,1,float16,float16,0,0.01462399959564209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,1,128,1,float16,fp8,0,0.017114666601022083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,2,128,1,float16,float16,0,0.014730667074521383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,2,128,1,float16,fp8,0,0.01714133347074191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,4,128,1,float16,float16,0,0.015194666882356008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,2,128,1,float16,float16,0,0.013738666971524557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,12,4,128,1,float16,fp8,0,0.016869333883126576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,12,128,1,float16,float16,0,0.012847999731699625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,12,128,1,float16,fp8,0,0.013237333546082178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,1,128,1,float16,float16,0,0.01379199946920077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,1,128,1,float16,fp8,0,0.015834666788578033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,1,128,1,float16,float16,0,0.013829333086808523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,2,128,1,float16,fp8,0,0.015696000307798386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,4,128,1,float16,float16,0,0.013861333330472311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,2,128,1,float16,fp8,0,0.015781333049138386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,12,4,128,1,float16,fp8,0,0.015930666277805965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,4,128,1,float16,float16,0,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,12,128,1,float16,float16,0,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,12,128,1,float16,fp8,0,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,1,128,1,float16,fp8,0,0.01569066693385442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,2,128,1,float16,float16,0,0.01357866699496905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,12,4,128,1,float16,fp8,0,0.01565333331624667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,1,128,1,float16,float16,0,0.06459199885527293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,2,128,1,float16,float16,0,0.08913066983222961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,1,128,1,float16,fp8,0,0.07209066549936931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,2,128,1,float16,fp8,0,0.09197333455085754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,4,128,1,float16,float16,0,0.12194133798281352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,12,128,1,float16,float16,0,0.09317333499590556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,12,128,1,float16,fp8,0,0.0942186713218689
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,12,4,128,1,float16,fp8,0,0.1423466702302297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,1,128,1,float16,float16,0,0.03105599929889043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,1,128,1,float16,fp8,0,0.03470933437347412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,12,128,1,float16,float16,0,0.048309331138928734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,2,128,1,float16,float16,0,0.042277331153551735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,2,128,1,float16,fp8,0,0.04504533112049103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,4,128,1,float16,float16,0,0.06543466448783875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,12,4,128,1,float16,fp8,0,0.06321600079536438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,12,128,1,float16,fp8,0,0.04268800218900045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,1,128,1,float16,float16,0,0.02207999924818675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,4,128,1,float16,fp8,0,0.034901333351929985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,1,128,1,float16,fp8,0,0.025349333882331848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,12,128,1,float16,fp8,0,0.025536000728607178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,2,128,1,float16,float16,0,0.024282666544119518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,2,128,1,float16,fp8,0,0.027797333896160126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,12,4,128,1,float16,float16,0,0.029359998802344005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,12,128,1,float16,float16,0,0.02420266717672348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,1,128,1,float16,float16,0,0.01563199982047081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,1,128,1,float16,fp8,0,0.018005333840847015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,2,128,1,float16,float16,0,0.01647466669480006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,2,128,1,float16,fp8,0,0.019274666905403137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,4,128,1,float16,float16,0,0.019023999571800232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,1,128,1,float16,fp8,0,0.017450666675964992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,12,4,128,1,float16,fp8,0,0.022261333962281544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,2,128,1,float16,fp8,0,0.017701332767804463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,12,128,1,float16,float16,0,0.017786666750907898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,4,128,1,float16,fp8,0,0.018853332847356796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,1,128,1,float16,float16,0,0.014922666052977243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,12,128,1,float16,fp8,0,0.01860800012946129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,2,128,1,float16,float16,0,0.014959999670584997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,12,4,128,1,float16,float16,0,0.01579733317097028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,12,128,1,float16,float16,0,0.013045333325862885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,12,128,1,float16,fp8,0,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,1,128,1,float16,float16,0,0.014650666465361914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,12,128,1,float16,float16,0,0.012741333494583765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,1,128,1,float16,fp8,0,0.01709866647919019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,2,128,1,float16,float16,0,0.014469332993030548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,2,128,1,float16,fp8,0,0.017242666333913803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,4,128,1,float16,float16,0,0.014645333091417948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,12,4,128,1,float16,fp8,0,0.017317333569129307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,12,128,1,float16,fp8,0,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,1,128,1,float16,float16,0,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,1,128,1,float16,fp8,0,0.015583999454975128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,2,128,1,float16,float16,0,0.013717333475748697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,2,128,1,float16,fp8,0,0.01571200042963028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,4,128,1,float16,float16,0,0.01369599997997284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,12,4,128,1,float16,fp8,0,0.015919999529918034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,12,128,1,float16,float16,0,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,12,128,1,float16,fp8,0,0.013045333325862885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,1,128,1,float16,float16,0,0.014682666709025701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,1,128,1,float16,fp8,0,0.015573333948850632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,2,128,1,float16,float16,0,0.013493333011865616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,2,128,1,float16,fp8,0,0.015722667177518208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,4,128,1,float16,float16,0,0.013359999905029932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,12,4,128,1,float16,fp8,0,0.015562667200962702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,12,128,1,float16,float16,0,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,12,128,1,float16,fp8,0,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,4,128,1,float16,float16,0,0.013376000026861826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,1,128,1,float16,float16,0,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,1,128,1,float16,fp8,0,0.015205333630243937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,2,128,1,float16,float16,0,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,2,128,1,float16,fp8,0,0.015263999501864115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,12,4,128,1,float16,fp8,0,0.015477333217859268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,1,128,1,float16,float16,0,0.035301332672437034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,1,128,1,float16,fp8,0,0.04008000095685323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,2,128,1,float16,float16,0,0.04249600072701772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,2,128,1,float16,fp8,0,0.044256001710891724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,4,128,1,float16,float16,0,0.07177599767843883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,12,128,1,float16,float16,0,0.0672266681989034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,12,4,128,1,float16,fp8,0,0.061936000982920326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,12,128,1,float16,fp8,0,0.06225599845250448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,1,128,1,float16,float16,0,0.021898667017618816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,1,128,1,float16,fp8,0,0.025125332176685333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,2,128,1,float16,float16,0,0.023930666347344715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,2,128,1,float16,fp8,0,0.02792000025510788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,4,128,1,float16,float16,0,0.032416000962257385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,12,4,128,1,float16,fp8,0,0.03458133339881897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,12,128,1,float16,float16,0,0.033546666304270424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,12,128,1,float16,fp8,0,0.035045333206653595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,1,128,1,float16,float16,0,0.015344000111023584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,1,128,1,float16,fp8,0,0.017562666287024815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,2,128,1,float16,float16,0,0.016389333953460056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,2,128,1,float16,fp8,0,0.019306667149066925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,4,128,1,float16,float16,0,0.02035733312368393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,12,4,128,1,float16,fp8,0,0.02197333425283432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,12,128,1,float16,float16,0,0.02143999934196472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,12,128,1,float16,fp8,0,0.022074667116006214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,1,128,1,float16,float16,0,0.014831999937693277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,1,128,1,float16,fp8,0,0.01682666689157486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,2,128,1,float16,float16,0,0.014576000471909841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,2,128,1,float16,fp8,0,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,4,128,1,float16,float16,0,0.014474666366974512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,12,4,128,1,float16,fp8,0,0.015216000378131866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,12,128,1,float16,float16,0,0.016271999726692837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,12,128,1,float16,fp8,0,0.016800000021855038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,1,128,1,float16,float16,0,0.014346666634082794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,1,128,1,float16,fp8,0,0.016719999412695568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,2,128,1,float16,float16,0,0.014303999642531076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,2,128,1,float16,fp8,0,0.016869333883126576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,4,128,1,float16,float16,0,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,12,4,128,1,float16,fp8,0,0.013477332890033722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,12,128,1,float16,float16,0,0.012826666235923767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,12,128,1,float16,fp8,0,0.01369599997997284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,1,128,1,float16,float16,0,0.013797332843144735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,1,128,1,float16,fp8,0,0.01579733317097028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,2,128,1,float16,float16,0,0.013738666971524557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,2,128,1,float16,fp8,0,0.015728000551462173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,4,128,1,float16,float16,0,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,12,4,128,1,float16,fp8,0,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,12,128,1,float16,float16,0,0.012624000509579977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,12,128,1,float16,fp8,0,0.013007999708255133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,1,128,1,float16,float16,0,0.013642666240533194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,1,128,1,float16,fp8,0,0.015450666348139444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,12,128,1,float16,fp8,0,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,2,128,1,float16,float16,0,0.013541333377361298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,2,128,1,float16,fp8,0,0.015615999698638916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,4,128,1,float16,float16,0,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,12,4,128,1,float16,fp8,0,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,12,128,1,float16,float16,0,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,1,128,1,float16,float16,0,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,1,128,1,float16,fp8,0,0.01524266724785169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,12,128,1,float16,fp8,0,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,2,128,1,float16,float16,0,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,2,128,1,float16,fp8,0,0.015178666760524115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,4,128,1,float16,float16,0,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,2,128,1,float16,fp8,0,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,12,4,128,1,float16,fp8,0,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,12,128,1,float16,float16,0,0.01163200040658315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,1,128,1,float16,float16,0,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,1,128,1,float16,fp8,0,0.015141333142916361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,2,128,1,float16,float16,0,0.012949333836634954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,4,128,1,float16,float16,0,0.01156266654531161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,0,0.019674666225910187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,12,4,128,1,float16,fp8,0,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,0,0.021226666867733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,0,0.02605333427588145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,0,0.027637332677841187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,0,0.038917332887649536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,12,12,128,1,float16,float16,0,0.04957866668701172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,0,0.01505600040157636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,0,0.041573333243529
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,12,12,128,1,float16,fp8,0,0.053157334526379905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,0,0.013781332721312841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,0,0.017530667285124462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,0,0.018437333405017853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,0,0.024234667420387268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,12,12,128,1,float16,float16,0,0.029845332105954487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,0,0.02569599946339925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,12,12,128,1,float16,fp8,0,0.032138665517171226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,0,0.016997333616018295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,0,0.012719999998807907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,0,0.013306666165590286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,0,0.012938667088747025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,0,0.013786666095256805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,0,0.018021332720915478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,12,12,128,1,float16,float16,0,0.019823999454577763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,12,12,128,1,float16,fp8,0,0.021082667013009388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,0,0.011850666254758835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,0,0.012666666259368261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,0,0.011994666109482447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,0,0.012522666404644648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,0,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,0,0.013349333157142004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,12,12,128,1,float16,float16,0,0.015919999529918034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,12,12,128,1,float16,fp8,0,0.01699200024207433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,0,0.011413333316644033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,0,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,0,0.011695999652147293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,0,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,0,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,0,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,0,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,12,12,128,1,float16,float16,0,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,12,12,128,1,float16,fp8,0,0.013023999830087027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,12,12,128,1,float16,float16,0,0.011482667177915573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,0,0.011194666226704916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,0,0.011701333026091257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,0,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,0,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,0,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,0,0.013034666577974955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,12,12,128,1,float16,fp8,0,0.012527999778588613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,0,0.01090666651725769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,0,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,0,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,0,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,0,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,0,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,12,12,128,1,float16,float16,0,0.011685332904259363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,12,12,128,1,float16,fp8,0,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,0,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,0,0.011002667248249054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,0,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,0,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,0,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,0,0.012608000387748083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,0,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,12,12,128,1,float16,float16,0,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,12,12,128,1,float16,fp8,0,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,0,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,0,0.01091733326514562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,0,0.011066666493813196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,0,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,float16,0,4.07368532816569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,1,128,1,float16,fp8,0,7.090272267659505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,fp8,0,4.105141321818034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,2,128,1,float16,fp8,0,7.199669520060222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,2,128,1,float16,float16,0,8.019055684407553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,4,128,1,float16,float16,0,8.256767908732096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,4,128,1,float16,fp8,0,7.998634974161784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,1,128,1,float16,float16,0,3.4913867314656577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,1,128,1,float16,fp8,0,3.574469248453776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,2,128,1,float16,float16,0,3.7730401357014975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,2,128,1,float16,fp8,0,3.6490612030029297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,8,1,128,1,float16,float16,0,13.712015787760416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,4,128,1,float16,float16,0,4.193973223368327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,4,128,1,float16,fp8,0,4.078037261962891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,float16,0,2.1086293856302896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,1,128,1,float16,float16,0,1.7962826093037922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,float16,0,4.1575625737508135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,fp8,0,2.4563093185424805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,8,8,128,1,float16,fp8,0,4.2384694417317705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,2,128,1,float16,float16,0,1.9816800753275554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,1,128,1,float16,fp8,0,3.9908320109049478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,4,128,1,float16,float16,0,2.075061321258545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,2,128,1,float16,fp8,0,1.8915467262268066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,4,128,1,float16,fp8,0,2.1209920247395835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,float16,0,2.1315040588378906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,8,8,128,1,float16,fp8,0,2.220911979675293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,float16,0,1.1099092960357666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,fp8,0,1.166927973429362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,2,128,1,float16,fp8,0,1.0285600026448567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,1,128,1,float16,fp8,0,1.0480053424835205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,2,128,1,float16,float16,0,1.001845359802246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,1,128,1,float16,float16,0,3.608405431111654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,4,128,1,float16,float16,0,1.0865493615468342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,4,128,1,float16,fp8,0,1.1376852989196777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,float16,0,1.109397331873576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,8,8,128,1,float16,fp8,0,1.1614293257395427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,1,128,1,float16,float16,0,3.981727917989095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,1,128,1,float16,fp8,0,4.162560145060222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,2,128,1,float16,float16,0,4.220970789591472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,float16,0,2.445802688598633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,fp8,0,2.492522716522217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,4,128,1,float16,float16,0,4.89796257019043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,1,128,1,float16,float16,0,2.2686986923217773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,4,128,1,float16,fp8,0,4.811029434204102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,1,128,1,float16,fp8,0,2.135279973347982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,2,128,1,float16,float16,0,2.2227253913879395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,8,2,128,1,float16,fp8,0,8.300677617390951
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,2,128,1,float16,fp8,0,2.2045013109842935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,4,128,1,float16,float16,0,2.4218880335489907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,float16,0,1.2765920162200928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,float16,0,2.462597370147705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,4,128,1,float16,fp8,0,2.78382937113444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,8,8,128,1,float16,fp8,0,2.485402743021647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,1,128,1,float16,float16,0,1.0475359757741292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,1,128,1,float16,fp8,0,1.1266187032063801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,fp8,0,2.3611413637797036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,2,128,1,float16,float16,0,1.1420693397521973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,2,128,1,float16,fp8,0,1.1586399873097737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,4,128,1,float16,float16,0,1.263983964920044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,4,128,1,float16,fp8,0,1.2855520248413086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,float16,0,0.9959626992543539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,float16,0,1.2743199666341145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,8,8,128,1,float16,fp8,0,1.3347360293070476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,fp8,0,0.7241333325703939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,1,128,1,float16,float16,0,0.5612426598866781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,4,128,1,float16,float16,0,0.6696960131327311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,1,128,1,float16,fp8,0,0.6141279935836792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,2,128,1,float16,float16,0,0.6049866676330566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,2,128,1,float16,fp8,0,0.8588053385416666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,4,128,1,float16,fp8,0,0.7117493152618408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,float16,0,0.6905653476715088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,8,8,128,1,float16,fp8,0,0.7275199890136719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,1,128,1,float16,float16,0,2.8179734547932944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,1,128,1,float16,fp8,0,2.957749366760254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,2,128,1,float16,float16,0,3.0570665995279946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,float16,0,1.789632002512614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,fp8,0,1.8583359718322754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,1,128,1,float16,float16,0,1.592938741048177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,4,128,1,float16,float16,0,3.4491306940714517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,4,128,1,float16,fp8,0,3.5522079467773438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,1,128,1,float16,fp8,0,1.5245973269144695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,2,128,1,float16,float16,0,1.6006186803181965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,2,128,1,float16,fp8,0,1.5607147216796875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,4,128,1,float16,float16,0,1.758885383605957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,8,2,128,1,float16,fp8,0,6.9465758005778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,4,128,1,float16,fp8,0,1.9375146230061848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,float16,0,0.9340319633483887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,float16,0,1.8036479949951172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,fp8,0,1.0173439979553223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,8,8,128,1,float16,fp8,0,1.8623305956522624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,1,128,1,float16,float16,0,0.760165294011434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,1,128,1,float16,fp8,0,0.8070986270904541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,2,128,1,float16,fp8,0,0.8497653007507324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,2,128,1,float16,float16,0,2.045695940653483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,fp8,0,0.9704266389211019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,4,128,1,float16,float16,0,0.9144426981608073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,4,128,1,float16,fp8,0,0.9723733266194662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,8,8,128,1,float16,float16,0,0.935103972752889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,float16,0,0.5132799943288168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,fp8,0,0.5446826616923014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,1,128,1,float16,float16,0,0.4068693319956462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,4,128,1,float16,fp8,0,0.5343253215154012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,1,128,1,float16,fp8,0,0.5233920017878214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,2,128,1,float16,float16,0,0.44230933984120685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,2,128,1,float16,fp8,0,0.4546186526616414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,4,128,1,float16,float16,0,0.5029439926147461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,float16,0,0.5125546852747599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,8,8,128,1,float16,fp8,0,0.5425386826197306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,1,128,1,float16,float16,0,3.646357218424479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,1,128,1,float16,fp8,0,3.8649867375691733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,2,128,1,float16,fp8,0,3.9463628133138022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,float16,0,2.373701254526774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,fp8,0,2.4296852747599282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,4,128,1,float16,float16,0,4.694944063822429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,1,128,1,float16,float16,0,1.8653066953023274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,4,128,1,float16,fp8,0,4.794053395589192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,1,128,1,float16,fp8,0,1.9725972811381023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,2,128,1,float16,float16,0,2.004746596018473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,2,128,1,float16,fp8,0,2.040090719858805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,4,128,1,float16,float16,0,2.334010601043701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,4,128,1,float16,fp8,0,2.722090721130371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,float16,0,1.2206453482309978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,float16,0,2.3693386713663735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,fp8,0,1.3703413009643555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,1,128,1,float16,float16,0,0.9640159606933594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,8,8,128,1,float16,fp8,0,2.414357344309489
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,2,128,1,float16,fp8,0,1.0355520248413086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,8,2,128,1,float16,float16,0,11.96340815226237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,1,128,1,float16,fp8,0,1.0018933614095051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,4,128,1,float16,fp8,0,1.2755839824676514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,2,128,1,float16,float16,0,1.063536008199056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,4,128,1,float16,float16,0,1.2107946872711182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,float16,0,1.2412959734598796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,1,128,1,float16,fp8,0,0.5495359897613525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,float16,0,0.6425013144810995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,8,8,128,1,float16,fp8,0,1.2843786875406902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,4,128,1,float16,float16,0,0.6381226778030396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,fp8,0,1.0985546906789143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,1,128,1,float16,float16,0,0.5062453349431356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,2,128,1,float16,float16,0,0.5582506656646729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,2,128,1,float16,fp8,0,0.5668533245722452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,4,128,1,float16,fp8,0,0.6856533686319987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,float16,0,0.6461759805679321
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,8,8,128,1,float16,fp8,0,0.685370683670044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,float16,0,0.36317865053812665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,fp8,0,0.3860960006713867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,1,128,1,float16,float16,0,0.2770400047302246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,1,128,1,float16,fp8,0,0.3028586705525716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,2,128,1,float16,float16,0,0.31037867069244385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,2,128,1,float16,fp8,0,0.32043200731277466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,4,128,1,float16,float16,0,0.3521973292032878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,4,128,1,float16,fp8,0,0.3744373321533203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,float16,0,0.36389867464701336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,8,8,128,1,float16,fp8,0,0.3906826575597127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,1,128,1,float16,float16,0,2.150949319203695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,1,128,1,float16,fp8,0,2.273301283518473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,2,128,1,float16,float16,0,2.357477347056071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,2,128,1,float16,fp8,0,2.3818079630533853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,float16,0,1.4693867365519206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,4,128,1,float16,float16,0,2.843045234680176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,fp8,0,1.748517354329427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,1,128,1,float16,float16,0,1.0921759605407715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,1,128,1,float16,fp8,0,1.165445327758789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,8,4,128,1,float16,fp8,0,3.0271145502726235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,2,128,1,float16,float16,0,1.2223947048187256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,2,128,1,float16,fp8,0,1.222256024678548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,4,128,1,float16,float16,0,1.4620320002237956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,4,128,1,float16,fp8,0,1.5944533348083496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,float16,0,1.4731520016988118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,1,128,1,float16,float16,0,0.5709120035171509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,float16,0,0.7697227001190186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,2,128,1,float16,float16,0,0.6442506710688273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,8,8,128,1,float16,fp8,0,1.5611786842346191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,fp8,0,0.82587202390035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,1,128,1,float16,fp8,0,0.6148586670557658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,2,128,1,float16,fp8,0,0.6453760067621866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,4,128,1,float16,fp8,0,0.807925303777059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,4,128,1,float16,float16,0,0.9026986757914225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,float16,0,0.7707306543986002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,8,8,128,1,float16,fp8,0,0.8147520224253336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,float16,0,0.4131466547648112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,fp8,0,0.4474506775538127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,1,128,1,float16,float16,0,0.3098613421122233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,1,128,1,float16,fp8,0,0.33459198474884033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,2,128,1,float16,float16,0,0.3507946729660034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,2,128,1,float16,fp8,0,0.5406239827473959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,4,128,1,float16,float16,0,0.4097546736399333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,4,128,1,float16,fp8,0,0.4347519874572754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,float16,0,0.4115413427352905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,8,8,128,1,float16,fp8,0,0.4471413294474284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,float16,0,0.24131733179092407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,fp8,0,0.2585600018501282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,1,128,1,float16,float16,0,0.1754080057144165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,1,128,1,float16,fp8,0,0.21556266148885092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,2,128,1,float16,float16,0,0.2005386749903361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,2,128,1,float16,fp8,0,0.20270399252573648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,4,128,1,float16,float16,0,0.23034133513768515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,4,128,1,float16,fp8,0,0.24709866444269815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,float16,0,0.2408693234125773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,8,8,128,1,float16,fp8,0,0.25867199897766113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,1,128,1,float16,float16,0,2.025141398111979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,2,128,1,float16,float16,0,2.3701440493265786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,2,128,1,float16,fp8,0,2.291445255279541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,float16,0,1.5138773918151855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,1,128,1,float16,fp8,0,5.218362808227539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,4,128,1,float16,float16,0,2.9128265380859375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,fp8,0,1.5926027297973633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,1,128,1,float16,float16,0,1.0494346618652344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,8,4,128,1,float16,fp8,0,3.119135856628418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,1,128,1,float16,fp8,0,1.1044267018636067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,2,128,1,float16,float16,0,1.1958879629770915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,2,128,1,float16,fp8,0,1.2013973395029705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,4,128,1,float16,fp8,0,1.6100373268127441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,float16,0,1.5020373662312825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,float16,0,0.7841173013051351
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,4,128,1,float16,float16,0,2.7854185104370117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,1,128,1,float16,fp8,0,0.5731306473414103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,8,8,128,1,float16,fp8,0,1.6258773803710938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,fp8,0,0.8606133460998535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,1,128,1,float16,float16,0,0.5347679853439331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,2,128,1,float16,float16,0,0.6240106821060181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,2,128,1,float16,fp8,0,0.6208426554997762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,4,128,1,float16,float16,0,0.7568853696187338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,float16,0,0.7945013046264648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,8,128,1,float16,fp8,0,0.8706506888071696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,8,4,128,1,float16,fp8,0,1.586367925008138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,float16,0,0.4203626712163289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,fp8,0,0.4573599894841512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,4,128,1,float16,float16,0,0.4094933271408081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,1,128,1,float16,float16,0,0.2874666651089986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,1,128,1,float16,fp8,0,0.31009066104888916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,2,128,1,float16,float16,0,0.33190399408340454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,2,128,1,float16,fp8,0,0.33881068229675293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,4,128,1,float16,fp8,0,0.6693759759267172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,float16,0,0.41864001750946045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,8,8,128,1,float16,fp8,0,0.45744001865386963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,float16,0,0.2349440058072408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,fp8,0,0.25541865825653076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,1,128,1,float16,float16,0,0.1583626667658488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,1,128,1,float16,fp8,0,0.16977600256601968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,2,128,1,float16,float16,0,0.19065600633621216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,2,128,1,float16,fp8,0,0.25358933210372925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,4,128,1,float16,float16,0,0.23147734006245932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,1,128,1,float16,float16,0,0.09332799911499023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,4,128,1,float16,fp8,0,0.24837332963943481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,float16,0,0.23469332853953043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,2,128,1,float16,fp8,0,0.11571199695269267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,8,8,128,1,float16,fp8,0,0.2563573320706685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,float16,0,0.14020267128944397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,fp8,0,0.15380799770355225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,1,128,1,float16,fp8,0,0.1169653336207072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,2,128,1,float16,float16,0,0.10819199681282043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,4,128,1,float16,float16,0,0.12833600242932638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,4,128,1,float16,fp8,0,0.13430399696032205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,float16,0,0.13871467113494873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,8,8,128,1,float16,fp8,0,0.15198933084805807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,1,128,1,float16,float16,0,1.2226719856262207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,2,128,1,float16,float16,0,1.4224212964375813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,1,128,1,float16,fp8,0,1.480288028717041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,2,128,1,float16,fp8,0,1.4359787305196126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,4,128,1,float16,float16,0,1.891109307607015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,float16,0,0.985706647237142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,1,128,1,float16,float16,0,0.6324853499730428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,8,4,128,1,float16,fp8,0,2.060922622680664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,fp8,0,1.0957973003387451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,1,128,1,float16,fp8,0,0.9163359800974528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,2,128,1,float16,float16,0,0.7416799863179525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,2,128,1,float16,fp8,0,0.7372106711069742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,float16,0,0.9788586298624674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,4,128,1,float16,float16,0,0.9557279745737711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,4,128,1,float16,fp8,0,1.0619893074035645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,1,128,1,float16,fp8,0,0.36050665378570557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,2,128,1,float16,float16,0,0.403658668200175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,8,8,128,1,float16,fp8,0,1.0913173357645671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,float16,0,0.5512746572494507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,fp8,0,0.5719466606775919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,1,128,1,float16,float16,0,0.3327626585960388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,2,128,1,float16,fp8,0,0.40139734745025635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,4,128,1,float16,float16,0,0.5027893384297689
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,4,128,1,float16,fp8,0,0.556170662244161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,float16,0,0.511247992515564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,1,128,1,float16,fp8,0,0.20198400815327963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,8,8,128,1,float16,fp8,0,0.5721333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,float16,0,0.284768005212148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,4,128,1,float16,float16,0,0.2792106668154399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,fp8,0,0.3094186584154765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,1,128,1,float16,float16,0,0.1834719975789388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,2,128,1,float16,float16,0,0.22172266244888306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,2,128,1,float16,fp8,0,0.22220265865325928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,4,128,1,float16,fp8,0,0.3025760054588318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,float16,0,0.2804746627807617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,8,8,128,1,float16,fp8,0,0.3104319969813029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,float16,0,0.1599573294321696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,fp8,0,0.18026133378346762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,1,128,1,float16,float16,0,0.10515200098355611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,1,128,1,float16,fp8,0,0.11749866604804993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,2,128,1,float16,float16,0,0.12473600109418233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,2,128,1,float16,fp8,0,0.1256533364454905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,4,128,1,float16,float16,0,0.15454399585723877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,4,128,1,float16,fp8,0,0.16794133186340332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,float16,0,0.15964800119400024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,8,8,128,1,float16,fp8,0,0.17773866653442383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,float16,0,0.09108799695968628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,fp8,0,0.10124799609184265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,1,128,1,float16,float16,0,0.06701866785685222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,float16,0,0.09037333726882935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,1,128,1,float16,fp8,0,0.07614399989446004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,2,128,1,float16,float16,0,0.07213866710662842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,2,128,1,float16,fp8,0,0.07850133379300435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,4,128,1,float16,float16,0,0.0828959991534551
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,4,128,1,float16,fp8,0,0.0946560005346934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,8,8,128,1,float16,fp8,0,0.10146666566530864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,1,128,1,float16,float16,0,1.2127947012583415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,1,128,1,float16,fp8,0,1.3086026509602864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,2,128,1,float16,float16,0,1.518133322397868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,2,128,1,float16,fp8,0,1.4877972602844238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,4,128,1,float16,fp8,0,2.319904009501139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,float16,0,1.077071984608968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,8,4,128,1,float16,float16,0,3.483637491861979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,fp8,0,1.1871413389841716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,1,128,1,float16,float16,0,0.6200053294499716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,1,128,1,float16,fp8,0,0.6742239793141683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,2,128,1,float16,float16,0,0.7760426998138428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,2,128,1,float16,fp8,0,0.7631893157958984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,4,128,1,float16,float16,0,1.0604426860809326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,4,128,1,float16,fp8,0,1.1826773484547932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,float16,0,0.574832002321879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,fp8,0,1.1856906414031982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,8,8,128,1,float16,float16,0,2.3751254081726074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,fp8,0,0.6179093519846598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,4,128,1,float16,float16,0,0.5533813238143921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,1,128,1,float16,float16,0,0.32730666796366376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,1,128,1,float16,fp8,0,0.3520266612370809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,2,128,1,float16,float16,0,0.41389866669972736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,2,128,1,float16,fp8,0,0.3985866705576579
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,4,128,1,float16,fp8,0,0.6151306629180908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,float16,0,0.6883306503295898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,float16,0,0.3042186697324117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,8,8,128,1,float16,fp8,0,0.6183573404947916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,fp8,0,0.339685320854187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,1,128,1,float16,float16,0,0.17932800451914468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,1,128,1,float16,fp8,0,0.19349332650502524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,2,128,1,float16,float16,0,0.22665599981943765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,fp8,0,0.3407520055770874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,2,128,1,float16,fp8,0,0.2220053275426229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,4,128,1,float16,float16,0,0.3144800066947937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,1,128,1,float16,fp8,0,0.10915733377138774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,4,128,1,float16,fp8,0,0.3285440007845561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,8,8,128,1,float16,float16,0,0.3025760054588318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,float16,0,0.16758400201797485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,fp8,0,0.18570667505264282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,1,128,1,float16,float16,0,0.0986293355623881
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,2,128,1,float16,float16,0,0.1267466644446055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,2,128,1,float16,fp8,0,0.12781332929929098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,fp8,0,0.18753600120544434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,4,128,1,float16,float16,0,0.16154666741689047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,4,128,1,float16,fp8,0,0.18186666568120322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,8,8,128,1,float16,float16,0,0.16780267159144083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,float16,0,0.09629333019256592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,2,128,1,float16,fp8,0,0.0748586654663086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,fp8,0,0.10710933804512024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,1,128,1,float16,float16,0,0.06163733204205831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,1,128,1,float16,fp8,0,0.0690719981988271
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,2,128,1,float16,float16,0,0.07001600166161855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,4,128,1,float16,float16,0,0.08764800429344177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,4,128,1,float16,fp8,0,0.09359467029571533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,float16,0,0.09703999757766724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,8,8,128,1,float16,fp8,0,0.10680533448855083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,float16,0,0.04864533245563507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,fp8,0,0.054117331902186074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,1,128,1,float16,float16,0,0.03833066672086716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,float16,0,0.04866133133570353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,1,128,1,float16,fp8,0,0.04326933125654856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,8,128,1,float16,fp8,0,0.05463466544946035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,2,128,1,float16,float16,0,0.04167466859022776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,2,128,1,float16,fp8,0,0.046037331223487854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,4,128,1,float16,float16,0,0.04692266881465912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,8,4,128,1,float16,fp8,0,0.05319466690222422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,1,128,1,float16,float16,0,0.8519413471221924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,1,128,1,float16,fp8,0,0.8778666655222574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,2,128,1,float16,float16,0,1.7064533233642578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,2,128,1,float16,fp8,0,1.007530689239502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,float16,0,0.7929226557413737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,4,128,1,float16,float16,0,1.5070346196492512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,fp8,0,0.8785066604614258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,8,4,128,1,float16,fp8,0,1.6440480550130208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,1,128,1,float16,float16,0,0.4416053295135498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,1,128,1,float16,fp8,0,0.45446932315826416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,2,128,1,float16,float16,0,0.5770080089569092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,2,128,1,float16,fp8,0,0.52292799949646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,4,128,1,float16,float16,0,0.7698506514231364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,4,128,1,float16,fp8,0,0.838805357615153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,float16,0,0.7984000047047933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,float16,0,0.4162079890569051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,8,8,128,1,float16,fp8,0,0.8690719604492188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,fp8,0,0.4601600170135498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,1,128,1,float16,float16,0,0.23693867524464926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,1,128,1,float16,fp8,0,0.24093333880106607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,2,128,1,float16,float16,0,0.30239466826121014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,2,128,1,float16,fp8,0,0.2803679903348287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,4,128,1,float16,float16,0,0.4119733174641927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,4,128,1,float16,fp8,0,0.4317813316980998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,float16,0,0.4150773286819458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,8,8,128,1,float16,fp8,0,0.4535520076751709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,float16,0,0.2172159949938456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,fp8,0,0.2442773381868998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,1,128,1,float16,float16,0,0.1304213305314382
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,1,128,1,float16,fp8,0,0.13547733426094055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,2,128,1,float16,float16,0,0.16434666514396667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,2,128,1,float16,fp8,0,0.15737066666285196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,4,128,1,float16,float16,0,0.21412799755732217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,4,128,1,float16,fp8,0,0.23522132635116577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,float16,0,0.21782400210698447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,8,8,128,1,float16,fp8,0,0.2441813349723816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,float16,0,0.12306666374206543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,fp8,0,0.13808000087738037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,1,128,1,float16,float16,0,0.07235200206438701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,1,128,1,float16,fp8,0,0.07534400125344594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,2,128,1,float16,float16,0,0.09000000357627869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,float16,0,0.07211733361085255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,2,128,1,float16,fp8,0,0.08551999926567078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,4,128,1,float16,float16,0,0.11743999520937602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,4,128,1,float16,fp8,0,0.12583466370900473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,float16,0,0.12329600254694621
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,8,8,128,1,float16,fp8,0,0.13690132896105447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,fp8,0,0.07889066636562347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,1,128,1,float16,float16,0,0.0454720010360082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,1,128,1,float16,fp8,0,0.04854399959246317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,2,128,1,float16,float16,0,0.0517439991235733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,2,128,1,float16,fp8,0,0.05351466437180837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,4,128,1,float16,float16,0,0.060559997955958046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,1,128,1,float16,float16,0,0.032261334359645844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,4,128,1,float16,fp8,0,0.06932266553243001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,float16,0,0.07031466563542683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,8,8,128,1,float16,fp8,0,0.07799999912579854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,float16,0,0.03941333293914795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,fp8,0,0.04413333535194397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,1,128,1,float16,fp8,0,0.03439466655254364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,2,128,1,float16,float16,0,0.03510933369398117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,2,128,1,float16,fp8,0,0.03682666768630346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,4,128,1,float16,float16,0,0.03873066604137421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,4,128,1,float16,fp8,0,0.04311466713746389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,float16,0,0.03930133332808813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,8,8,128,1,float16,fp8,0,0.0439626673857371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,1,128,1,float16,float16,0,0.9284906387329102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,1,128,1,float16,fp8,0,0.97597336769104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,2,128,1,float16,float16,0,1.2437386512756348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,2,128,1,float16,fp8,0,1.1708746751149495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,float16,0,0.9256746768951416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,fp8,0,1.0345760186513264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,1,128,1,float16,float16,0,0.47512535254160565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,1,128,1,float16,fp8,0,0.5032480160395304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,4,128,1,float16,float16,0,1.7865974108378093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,2,128,1,float16,float16,0,0.6296960115432739
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,8,4,128,1,float16,fp8,0,1.9970666567484539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,2,128,1,float16,fp8,0,0.5977813402811686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,4,128,1,float16,float16,0,0.9176106452941895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,4,128,1,float16,fp8,0,1.0267199675242107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,float16,0,0.9264372984568278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,float16,0,0.494704008102417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,1,128,1,float16,float16,0,0.2525599996248881
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,2,128,1,float16,float16,0,0.32798399527867633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,2,128,1,float16,fp8,0,0.31699200471242267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,4,128,1,float16,float16,0,0.46964800357818604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,4,128,1,float16,fp8,0,0.5236639976501465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,fp8,0,0.5569599866867065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,8,8,128,1,float16,fp8,0,1.029040018717448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,1,128,1,float16,fp8,0,0.2662293314933777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,float16,0,0.4994293451309204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,float16,0,0.26146133740743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,2,128,1,float16,float16,0,0.18198933204015097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,8,8,128,1,float16,fp8,0,0.5531306664148966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,fp8,0,0.2922240098317464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,4,128,1,float16,float16,0,0.2551893393198649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,4,128,1,float16,fp8,0,0.28378132979075116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,1,128,1,float16,float16,0,0.13763200243314108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,1,128,1,float16,fp8,0,0.1481013298034668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,2,128,1,float16,fp8,0,0.17458132902781168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,float16,0,0.2608693242073059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,8,8,128,1,float16,fp8,0,0.29419199625651044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,float16,0,0.1386293371518453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,fp8,0,0.16039466857910156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,1,128,1,float16,float16,0,0.07708266874154408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,1,128,1,float16,fp8,0,0.08533866206804912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,2,128,1,float16,float16,0,0.09858133395512898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,2,128,1,float16,fp8,0,0.09876267115275066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,4,128,1,float16,float16,0,0.13775466879208884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,1,128,1,float16,float16,0,0.04498666524887085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,4,128,1,float16,fp8,0,0.1499733328819275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,float16,0,0.13961600263913473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,8,8,128,1,float16,fp8,0,0.16022400061289468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,float16,0,0.07774933179219563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,fp8,0,0.09153599540392558
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,1,128,1,float16,fp8,0,0.05113600194454193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,2,128,1,float16,fp8,0,0.05513600011666616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,float16,0,0.0372533326347669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,2,128,1,float16,float16,0,0.05141866703828176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,4,128,1,float16,float16,0,0.0687306672334671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,4,128,1,float16,fp8,0,0.07147199908892314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,float16,0,0.08014399806658427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,8,8,128,1,float16,fp8,0,0.09044800202051799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,fp8,0,0.04189866781234741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,1,128,1,float16,float16,0,0.02752000093460083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,1,128,1,float16,fp8,0,0.029968000948429108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,2,128,1,float16,float16,0,0.030495998760064442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,2,128,1,float16,fp8,0,0.03340800106525421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,fp8,0,0.034245334565639496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,1,128,1,float16,float16,0,0.02497600018978119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,4,128,1,float16,float16,0,0.03631466627120972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,4,128,1,float16,fp8,0,0.04045333216587702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,2,128,1,float16,fp8,0,0.02958400050799052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,float16,0,0.039061332742373146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,8,8,128,1,float16,fp8,0,0.04177600145339966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,float16,0,0.029626667499542236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,1,128,1,float16,fp8,0,0.027402666707833607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,2,128,1,float16,float16,0,0.026757332185904186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,4,128,1,float16,float16,0,0.02945599953333537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,4,128,1,float16,fp8,0,0.03322133421897888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,float16,0,0.029461334149042766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,8,8,128,1,float16,fp8,0,0.03453866640726725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,1,128,1,float16,float16,0,0.6204479932785034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,1,128,1,float16,fp8,0,0.7017652988433838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,1,128,1,float16,float16,0,0.3216906587282817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,2,128,1,float16,float16,0,0.9350186983744303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,1,128,1,float16,fp8,0,0.36345601081848145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,fp8,0,0.9634400208791097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,2,128,1,float16,float16,0,0.4673173427581787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,2,128,1,float16,fp8,0,0.8888586362202963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,2,128,1,float16,fp8,0,0.45877333482106525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,float16,0,0.8009280363718668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,4,128,1,float16,float16,0,1.4806826909383137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,4,128,1,float16,float16,0,0.7608853181203207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,8,4,128,1,float16,fp8,0,1.755184014638265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,4,128,1,float16,fp8,0,0.8948372999827067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,float16,0,0.41516268253326416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,float16,0,0.7957599957784017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,8,8,128,1,float16,fp8,0,0.9624319871266683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,fp8,0,0.4975893497467041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,1,128,1,float16,fp8,0,0.1939786672592163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,fp8,0,0.49713067213694256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,4,128,1,float16,float16,0,0.38966933886210126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,float16,0,0.2164319952329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,2,128,1,float16,float16,0,0.24554133415222168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,fp8,0,0.25674132506052655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,2,128,1,float16,fp8,0,0.2428426742553711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,2,128,1,float16,float16,0,0.1400266687075297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,1,128,1,float16,fp8,0,0.10895466804504395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,1,128,1,float16,float16,0,0.09656533598899841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,8,128,1,float16,float16,0,0.4140586853027344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,2,128,1,float16,fp8,0,0.136272003253301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,1,128,1,float16,float16,0,0.17246933778127035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,float16,0,0.21475734313329062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,4,128,1,float16,float16,0,0.2048906683921814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,8,4,128,1,float16,fp8,0,0.4609493414560954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,4,128,1,float16,fp8,0,0.2440426747004191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,float16,0,0.11922666430473328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,8,8,128,1,float16,fp8,0,0.2613813281059265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,fp8,0,0.14594667156537375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,1,128,1,float16,float16,0,0.05207466582457224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,1,128,1,float16,fp8,0,0.05981333553791046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,2,128,1,float16,float16,0,0.07825066645940144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,2,128,1,float16,fp8,0,0.07845866680145264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,4,128,1,float16,float16,0,0.11540800333023071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,4,128,1,float16,fp8,0,0.13316800196965536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,float16,0,0.11891733606656392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,8,8,128,1,float16,fp8,0,0.14433599511782327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,float16,0,0.06606400012969971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,fp8,0,0.07691733539104462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,1,128,1,float16,float16,0,0.03258133431275686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,1,128,1,float16,fp8,0,0.03800000001986822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,4,128,1,float16,fp8,0,0.06259199976921082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,2,128,1,float16,float16,0,0.038165333370367684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,2,128,1,float16,fp8,0,0.04321066538492838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,4,128,1,float16,float16,0,0.05574933191140493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,float16,0,0.06623466809590657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,8,8,128,1,float16,fp8,0,0.07695466776688893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,float16,0,0.02940266579389572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,fp8,0,0.036490666369597115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,4,128,1,float16,fp8,0,0.03634133438269297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,1,128,1,float16,float16,0,0.020981334149837494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,1,128,1,float16,fp8,0,0.024735999604066212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,2,128,1,float16,float16,0,0.023317334552605946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,2,128,1,float16,fp8,0,0.028309332827727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,4,128,1,float16,float16,0,0.02870933214823405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,float16,0,0.029418667157491047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,8,8,128,1,float16,fp8,0,0.036576000352700554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,2,128,1,float16,fp8,0,0.02474133421977361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,float16,0,0.023546665906906128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,fp8,0,0.029520000020662945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,float16,0,0.023423999547958374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,1,128,1,float16,float16,0,0.01915733392039935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,1,128,1,float16,fp8,0,0.02316266546646754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,2,128,1,float16,float16,0,0.020981334149837494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,4,128,1,float16,float16,0,0.023215999205907185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,4,128,1,float16,fp8,0,0.029258665939172108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,8,8,128,1,float16,fp8,0,0.029850666721661884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,2,128,1,float16,fp8,0,0.022858666876951855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,float16,0,0.020725333442290623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,fp8,0,0.02606933315594991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,float16,0,0.020799999435742695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,1,128,1,float16,float16,0,0.018981333822011948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,1,128,1,float16,fp8,0,0.0215786670645078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,2,128,1,float16,float16,0,0.019610666980346043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,4,128,1,float16,float16,0,0.02062400057911873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,4,128,1,float16,fp8,0,0.025920001169045765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,8,8,128,1,float16,fp8,0,0.02622399975856145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,1,128,1,float16,float16,0,0.27380800247192383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,1,128,1,float16,fp8,0,0.31600000460942584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,2,128,1,float16,float16,0,0.42477333545684814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,2,128,1,float16,fp8,0,0.4138400157292684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,1,128,1,float16,fp8,0,0.17057067155838013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,4,128,1,float16,float16,0,0.7127839724222819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,float16,0,0.39292800426483154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,1,128,1,float16,float16,0,0.14752533038457236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,2,128,1,float16,float16,0,0.21845332781473795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,2,128,1,float16,fp8,0,0.2207253376642863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,fp8,0,0.45810667673746747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,4,128,1,float16,fp8,0,0.43427733580271405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,fp8,0,0.4603573481241862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,8,4,128,1,float16,fp8,0,0.8484106858571371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,4,128,1,float16,float16,0,0.36295998096466064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,8,8,128,1,float16,float16,0,0.39410666624705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,float16,0,0.2039573391278585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,fp8,0,0.2391306757926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,1,128,1,float16,fp8,0,0.09564800063769023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,1,128,1,float16,float16,0,0.08386133114496867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,float16,0,0.20401066541671753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,8,128,1,float16,fp8,0,0.23876800139745077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,2,128,1,float16,fp8,0,0.12358933687210083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,2,128,1,float16,float16,0,0.12638399998346964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,4,128,1,float16,float16,0,0.19286932547887167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,float16,0,0.11276800433794658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,8,4,128,1,float16,fp8,0,0.23070400953292847
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,fp8,0,0.133925328652064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,1,128,1,float16,float16,0,0.043231998880704246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,1,128,1,float16,fp8,0,0.05065600077311198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,2,128,1,float16,float16,0,0.07145600020885468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,2,128,1,float16,fp8,0,0.06954666475454967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,4,128,1,float16,float16,0,0.1030613382657369
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,4,128,1,float16,fp8,0,0.12804800271987915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,float16,0,0.11231999595959981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,8,8,128,1,float16,fp8,0,0.1339040001233419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,float16,0,0.06108266611893972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,fp8,0,0.06851199766000111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,1,128,1,float16,float16,0,0.02741333345572154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,1,128,1,float16,fp8,0,0.03218133250872294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,fp8,0,0.06858666737874348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,2,128,1,float16,float16,0,0.03249600032965342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,2,128,1,float16,fp8,0,0.03742400060097376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,4,128,1,float16,float16,0,0.04613333443800608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,4,128,1,float16,fp8,0,0.05499200026194254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,8,8,128,1,float16,float16,0,0.061434666315714516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,float16,0,0.026250667870044708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,fp8,0,0.03368533402681351
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,4,128,1,float16,float16,0,0.025983999172846477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,1,128,1,float16,float16,0,0.018506667266289394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,1,128,1,float16,fp8,0,0.022154666483402252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,2,128,1,float16,float16,0,0.020501332978407543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,2,128,1,float16,fp8,0,0.024293333292007446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,fp8,0,0.02658133457104365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,4,128,1,float16,fp8,0,0.032298666735490165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,float16,0,0.027045334378878277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,2,128,1,float16,float16,0,0.017887999614079792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,2,128,1,float16,fp8,0,0.021002667645613354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,8,8,128,1,float16,fp8,0,0.03349333256483078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,float16,0,0.02089600016673406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,1,128,1,float16,float16,0,0.016176000237464905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,1,128,1,float16,fp8,0,0.01887999971707662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,4,128,1,float16,float16,0,0.020341333001852036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,fp8,0,0.023061332603295643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,4,128,1,float16,fp8,0,0.025429333249727886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,float16,0,0.020746666938066483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,8,8,128,1,float16,fp8,0,0.026320000489552815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,float16,0,0.01820266619324684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,1,128,1,float16,float16,0,0.01617066686352094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,1,128,1,float16,fp8,0,0.01859733338157336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,2,128,1,float16,float16,0,0.0163680004576842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,2,128,1,float16,fp8,0,0.01903466631968816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,4,128,1,float16,float16,0,0.017935999979575474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,1,128,1,float16,float16,0,0.015365333606799444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,4,128,1,float16,fp8,0,0.02178666740655899
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,float16,0,0.017749333133300144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,8,8,128,1,float16,fp8,0,0.022997332115968067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,float16,0,0.015685333559910457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,fp8,0,0.019445333629846573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,1,128,1,float16,fp8,0,0.018378666291634243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,2,128,1,float16,float16,0,0.01569066693385442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,2,128,1,float16,fp8,0,0.017781333376963932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,4,128,1,float16,float16,0,0.01621866722901662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,4,128,1,float16,fp8,0,0.019194666296243668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,float16,0,0.015728000551462173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,8,8,128,1,float16,fp8,0,0.019199999670187633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,1,128,1,float16,float16,0,0.14260266224543253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,1,128,1,float16,fp8,0,0.15095466375350952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,2,128,1,float16,float16,0,0.22451732556025186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,2,128,1,float16,fp8,0,0.2096959948539734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,float16,0,0.2023520072301229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,4,128,1,float16,float16,0,0.3556106487909953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,fp8,0,0.22394132614135742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,1,128,1,float16,float16,0,0.07746666669845581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,8,4,128,1,float16,fp8,0,0.4161546627680461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,1,128,1,float16,fp8,0,0.08545066912968953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,2,128,1,float16,float16,0,0.12035733461380005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,2,128,1,float16,fp8,0,0.11798399686813354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,4,128,1,float16,float16,0,0.18807466824849448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,float16,0,0.19951466719309488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,4,128,1,float16,fp8,0,0.21928000450134277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,float16,0,0.11031466722488403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,8,8,128,1,float16,fp8,0,0.22406399250030518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,fp8,0,0.12343999743461609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,1,128,1,float16,float16,0,0.03951466580231985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,1,128,1,float16,fp8,0,0.04473066826661428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,2,128,1,float16,float16,0,0.06833066542943318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,2,128,1,float16,fp8,0,0.06438399851322174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,4,128,1,float16,float16,0,0.10220799843470256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,4,128,1,float16,fp8,0,0.11589333415031433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,float16,0,0.10976533095041911
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,8,8,128,1,float16,fp8,0,0.12337066729863484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,float16,0,0.05888533095518748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,fp8,0,0.06249066690603892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,1,128,1,float16,float16,0,0.02476799984773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,1,128,1,float16,fp8,0,0.028160000840822857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,2,128,1,float16,float16,0,0.030229332546393078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,2,128,1,float16,fp8,0,0.03330666571855545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,4,128,1,float16,float16,0,0.04628799855709076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,4,128,1,float16,fp8,0,0.04868266483147939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,float16,0,0.05945600072542826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,8,8,128,1,float16,fp8,0,0.06262399752934773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,float16,0,0.02517866591612498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,fp8,0,0.029450667401154835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,1,128,1,float16,float16,0,0.016943999876578648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,1,128,1,float16,fp8,0,0.019434666881958645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,2,128,1,float16,float16,0,0.01911466692884763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,2,128,1,float16,fp8,0,0.022405333817005157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,4,128,1,float16,fp8,0,0.029274667302767437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,4,128,1,float16,float16,0,0.024586667617162068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,float16,0,0.025034666061401367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,2,128,1,float16,float16,0,0.01646399994691213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,8,8,128,1,float16,fp8,0,0.029482667644818623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,float16,0,0.019509332875410717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,fp8,0,0.022458667556444805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,1,128,1,float16,float16,0,0.014869333555301031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,1,128,1,float16,fp8,0,0.017749333133300144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,2,128,1,float16,fp8,0,0.019493332753578823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,4,128,1,float16,float16,0,0.01932266727089882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,4,128,1,float16,fp8,0,0.022543999056021374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,float16,0,0.019093333433071773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,8,8,128,1,float16,fp8,0,0.022266666094462078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,float16,0,0.016303999970356624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,fp8,0,0.01899733394384384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,1,128,1,float16,float16,0,0.014661333213249842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,4,128,1,float16,fp8,0,0.01884799947341283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,1,128,1,float16,fp8,0,0.017664000391960144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,2,128,1,float16,float16,0,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,2,128,1,float16,fp8,0,0.017488000293572743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,4,128,1,float16,float16,0,0.016336000214020412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,1,128,1,float16,float16,0,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,float16,0,0.01661866654952367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,8,8,128,1,float16,fp8,0,0.01899733394384384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,2,128,1,float16,fp8,0,0.016016000260909397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,float16,0,0.01421333352724711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,fp8,0,0.01632533346613248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,1,128,1,float16,fp8,0,0.015418666104475657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,2,128,1,float16,float16,0,0.013829333086808523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,4,128,1,float16,float16,0,0.014250667144854864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,4,128,1,float16,fp8,0,0.016042667130629223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,1,128,1,float16,float16,0,0.013445333888133367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,float16,0,0.014245333770910898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,8,8,128,1,float16,fp8,0,0.01602666700879733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,4,128,1,float16,float16,0,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,float16,0,0.014186666657527288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,fp8,0,0.015664000064134598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,1,128,1,float16,fp8,0,0.015429332852363586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,2,128,1,float16,float16,0,0.013776000589132309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,2,128,1,float16,fp8,0,0.015520000209410986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,4,128,1,float16,fp8,0,0.015834666788578033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,float16,0,0.013957332819700241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,8,8,128,1,float16,fp8,0,0.01579733317097028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,1,128,1,float16,float16,0,0.07714133461316426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,2,128,1,float16,float16,0,0.12220799922943115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,1,128,1,float16,fp8,0,0.08548266688982646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,2,128,1,float16,fp8,0,0.11693333586057027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,float16,0,0.11499733726183574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,4,128,1,float16,fp8,0,0.218341330687205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,8,4,128,1,float16,float16,0,0.18672533830006918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,fp8,0,0.11990400155385335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,1,128,1,float16,float16,0,0.03930133332808813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,1,128,1,float16,fp8,0,0.04433066646258036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,2,128,1,float16,float16,0,0.06867200136184692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,2,128,1,float16,fp8,0,0.06308266520500183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,4,128,1,float16,float16,0,0.10169066985448201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,4,128,1,float16,fp8,0,0.11518399914105733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,float16,0,0.11411733428637187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,8,8,128,1,float16,fp8,0,0.12029866377512614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,2,128,1,float16,float16,0,0.030154667794704437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,float16,0,0.06303999821345012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,fp8,0,0.055258666475613914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,1,128,1,float16,float16,0,0.024266667664051056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,1,128,1,float16,fp8,0,0.02826133370399475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,2,128,1,float16,fp8,0,0.03309866786003113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,float16,0,0.027109332382678986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,4,128,1,float16,float16,0,0.042394667863845825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,1,128,1,float16,float16,0,0.01681600014368693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,4,128,1,float16,fp8,0,0.04872000217437744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,float16,0,0.062394668658574425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,8,8,128,1,float16,fp8,0,0.059690664211908974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,fp8,0,0.028853334486484528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,1,128,1,float16,fp8,0,0.019493332753578823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,2,128,1,float16,float16,0,0.01931200052301089
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,fp8,0,0.02882133424282074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,2,128,1,float16,fp8,0,0.022613334159056347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,4,128,1,float16,float16,0,0.024346667031447094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,4,128,1,float16,fp8,0,0.029029332101345062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,8,8,128,1,float16,float16,0,0.02718399961789449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,float16,0,0.017322666943073273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,fp8,0,0.01868266612291336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,1,128,1,float16,float16,0,0.015504000087579092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,1,128,1,float16,fp8,0,0.01810666670401891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,2,128,1,float16,float16,0,0.0162773331006368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,2,128,1,float16,fp8,0,0.01918399954835574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,4,128,1,float16,float16,0,0.019082666685183842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,4,128,1,float16,fp8,0,0.022250667214393616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,float16,0,0.017429333180189133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,8,8,128,1,float16,fp8,0,0.018816000471512478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,float16,0,0.014314666390419006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,fp8,0,0.01563199982047081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,1,128,1,float16,float16,0,0.01444799949725469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,1,128,1,float16,fp8,0,0.017418666432301205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,2,128,1,float16,float16,0,0.015135999768972397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,2,128,1,float16,fp8,0,0.017818666994571686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,fp8,0,0.013594667116800943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,4,128,1,float16,float16,0,0.016293333222468693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,4,128,1,float16,fp8,0,0.018917333334684372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,float16,0,0.014394666999578476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,8,8,128,1,float16,fp8,0,0.015381333728631338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,float16,0,0.012954667210578918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,1,128,1,float16,float16,0,0.013728000223636627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,float16,0,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,1,128,1,float16,fp8,0,0.015498666713635126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,2,128,1,float16,float16,0,0.014042666802803675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,2,128,1,float16,fp8,0,0.01720533271630605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,4,128,1,float16,float16,0,0.014101333916187286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,4,128,1,float16,fp8,0,0.016154666741689045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,8,8,128,1,float16,fp8,0,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,float16,0,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,fp8,0,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,1,128,1,float16,float16,0,0.013637332866589228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,float16,0,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,1,128,1,float16,fp8,0,0.015647999942302704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,2,128,1,float16,float16,0,0.013951999445756277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,2,128,1,float16,fp8,0,0.015749332805474598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,fp8,0,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,4,128,1,float16,float16,0,0.013744000345468521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,4,128,1,float16,fp8,0,0.01573866605758667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,8,8,128,1,float16,fp8,0,0.012624000509579977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,2,128,1,float16,fp8,0,0.015295999745527903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,float16,0,0.011695999652147293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,1,128,1,float16,float16,0,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,1,128,1,float16,fp8,0,0.015061333775520325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,2,128,1,float16,float16,0,0.013349333157142004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,4,128,1,float16,float16,0,0.01340266689658165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,4,128,1,float16,fp8,0,0.015450666348139444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,float16,0,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,8,8,128,1,float16,fp8,0,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,1,128,1,float16,float16,0,0.0384853333234787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,1,128,1,float16,fp8,0,0.04443199932575226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,2,128,1,float16,float16,0,0.06729599833488464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,2,128,1,float16,fp8,0,0.060821334520975746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,4,128,1,float16,float16,0,0.10637332995732625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,float16,0,0.07441600163777669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,fp8,0,0.07072000205516815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,8,4,128,1,float16,fp8,0,0.11229333281517029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,1,128,1,float16,float16,0,0.024501333634058636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,2,128,1,float16,fp8,0,0.03313600023587545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,1,128,1,float16,fp8,0,0.02824000020821889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,2,128,1,float16,float16,0,0.029637334247430164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,4,128,1,float16,float16,0,0.05089599887530009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,4,128,1,float16,fp8,0,0.04790933430194855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,float16,0,0.07411733269691467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,float16,0,0.032272001107533775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,8,8,128,1,float16,fp8,0,0.07205866773923238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,fp8,0,0.03473600000143051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,1,128,1,float16,float16,0,0.016949333250522614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,1,128,1,float16,fp8,0,0.01961600035429001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,2,128,1,float16,float16,0,0.019413333386182785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,2,128,1,float16,fp8,0,0.02250666668017705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,4,128,1,float16,float16,0,0.025770666698614757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,4,128,1,float16,fp8,0,0.028501334289709728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,float16,0,0.032645332316557564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,8,8,128,1,float16,fp8,0,0.03493333359559377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,float16,0,0.020741333564122517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,fp8,0,0.022069332500298817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,1,128,1,float16,float16,0,0.015509333461523056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,1,128,1,float16,fp8,0,0.017290666699409485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,2,128,1,float16,float16,0,0.016250666230916977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,2,128,1,float16,fp8,0,0.018981333822011948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,4,128,1,float16,float16,0,0.01709866647919019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,4,128,1,float16,fp8,0,0.01854933301607768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,float16,0,0.020687999824682873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,8,8,128,1,float16,fp8,0,0.021984001000722248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,float16,0,0.014671999961137772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,fp8,0,0.015135999768972397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,1,128,1,float16,float16,0,0.014378666877746582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,1,128,1,float16,fp8,0,0.016943999876578648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,2,128,1,float16,float16,0,0.01482133318980535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,2,128,1,float16,fp8,0,0.017349333812793095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,4,128,1,float16,float16,0,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,4,128,1,float16,fp8,0,0.015125333021084467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,float16,0,0.014639999717473984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,8,8,128,1,float16,fp8,0,0.015168000012636185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,float16,0,0.012815999488035837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,fp8,0,0.013663999736309052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,4,128,1,float16,fp8,0,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,1,128,1,float16,float16,0,0.014266667266686758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,1,128,1,float16,fp8,0,0.015615999698638916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,2,128,1,float16,float16,0,0.01441066712141037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,fp8,0,0.013114667187134424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,2,128,1,float16,fp8,0,0.01573866605758667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,4,128,1,float16,float16,0,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,float16,0,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,8,8,128,1,float16,fp8,0,0.013343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,float16,0,0.012421333541472753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,1,128,1,float16,float16,0,0.013845333208640417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,1,128,1,float16,fp8,0,0.01568000018596649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,2,128,1,float16,float16,0,0.013786666095256805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,2,128,1,float16,fp8,0,0.015520000209410986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,4,128,1,float16,float16,0,0.012698666503032049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,4,128,1,float16,fp8,0,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,float16,0,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,8,8,128,1,float16,fp8,0,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,2,128,1,float16,fp8,0,0.015706667055686314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,float16,0,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,fp8,0,0.012837332983811697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,float16,0,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,1,128,1,float16,float16,0,0.013280000537633896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,float16,0,0.011674666156371435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,1,128,1,float16,fp8,0,0.015237333873907724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,2,128,1,float16,float16,0,0.013466666142145792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,4,128,1,float16,float16,0,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,4,128,1,float16,fp8,0,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,8,8,128,1,float16,fp8,0,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,fp8,0,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,1,128,1,float16,float16,0,0.01312000056107839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,float16,0,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,1,128,1,float16,fp8,0,0.015210667004187902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,2,128,1,float16,float16,0,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,2,128,1,float16,fp8,0,0.015504000087579092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,4,128,1,float16,float16,0,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,4,128,1,float16,fp8,0,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,1,128,1,float16,fp8,0,0.028368001182874043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,8,8,128,1,float16,fp8,0,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,1,128,1,float16,float16,0,0.024085332949956257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,2,128,1,float16,float16,0,0.032325332363446556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,2,128,1,float16,fp8,0,0.03317866722742716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,4,128,1,float16,float16,0,0.05900266766548157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,float16,0,0.04596266647179922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,8,4,128,1,float16,fp8,0,0.059808000922203064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,fp8,0,0.04678933322429657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,1,128,1,float16,float16,0,0.016800000021855038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,1,128,1,float16,fp8,0,0.019445333629846573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,2,128,1,float16,float16,0,0.020794666061798733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,2,128,1,float16,fp8,0,0.021914665897687275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,4,128,1,float16,float16,0,0.03200533241033554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,4,128,1,float16,fp8,0,0.03448000053564707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,float16,0,0.04478933413823446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,8,8,128,1,float16,fp8,0,0.04693333307902018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,float16,0,0.027029333015282948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,fp8,0,0.028351999819278717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,1,128,1,float16,float16,0,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,1,128,1,float16,fp8,0,0.01720000058412552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,2,128,1,float16,float16,0,0.014479999740918478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,2,128,1,float16,fp8,0,0.015247999380032221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,4,128,1,float16,float16,0,0.020448000480731327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,4,128,1,float16,fp8,0,0.021941334009170532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,float16,0,0.02699733277161916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,8,8,128,1,float16,fp8,0,0.028431999186674755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,float16,0,0.01782400036851565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,fp8,0,0.01874133323629697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,1,128,1,float16,float16,0,0.014490666488806406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,4,128,1,float16,fp8,0,0.015333333363135656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,1,128,1,float16,fp8,0,0.016837333639462788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,fp8,0,0.018650667121013004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,2,128,1,float16,float16,0,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,fp8,0,0.013653332988421122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,2,128,1,float16,fp8,0,0.01349866638580958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,4,128,1,float16,float16,0,0.014346666634082794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,8,8,128,1,float16,float16,0,0.017952000101407368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,float16,0,0.013045333325862885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,1,128,1,float16,float16,0,0.013562666873137156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,1,128,1,float16,fp8,0,0.015429332852363586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,2,128,1,float16,float16,0,0.012666666259368261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,2,128,1,float16,fp8,0,0.013269333789745966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,fp8,0,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,4,128,1,float16,float16,0,0.012847999731699625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,4,128,1,float16,fp8,0,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,float16,0,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,8,8,128,1,float16,fp8,0,0.01341333364446958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,4,128,1,float16,float16,0,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,float16,0,0.012517333030700684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,float16,0,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,1,128,1,float16,float16,0,0.013594667116800943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,1,128,1,float16,fp8,0,0.015530666957298914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,2,128,1,float16,float16,0,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,2,128,1,float16,fp8,0,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,1,128,1,float16,fp8,0,0.01523200049996376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,4,128,1,float16,fp8,0,0.01309866706530253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,8,8,128,1,float16,fp8,0,0.01333333303531011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,float16,0,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,fp8,0,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,1,128,1,float16,float16,0,0.013157332936922709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,2,128,1,float16,float16,0,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,2,128,1,float16,fp8,0,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,4,128,1,float16,float16,0,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,4,128,1,float16,fp8,0,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,float16,0,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,8,8,128,1,float16,fp8,0,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,float16,0,0.011861333002646765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,fp8,0,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,1,128,1,float16,float16,0,0.012949333836634954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,1,128,1,float16,fp8,0,0.014991999914248785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,2,128,1,float16,float16,0,0.011391999820868174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,2,128,1,float16,fp8,0,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,4,128,1,float16,float16,0,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,4,128,1,float16,fp8,0,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,float16,0,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,8,8,128,1,float16,fp8,0,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,float16,0,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,fp8,0,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,1,128,1,float16,float16,0,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,1,128,1,float16,fp8,0,0.014933332800865173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,2,128,1,float16,float16,0,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,2,128,1,float16,fp8,0,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,4,128,1,float16,float16,0,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,4,128,1,float16,fp8,0,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,float16,0,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,0,0.02053333322207133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,8,8,128,1,float16,fp8,0,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,0,0.01940800001223882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,0,0.025829332570234936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,0,0.027503999571005504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,0,0.03872533390919367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,0,0.03668266783157984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,0,0.014896000425020853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,0,0.03993066648642222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,0,0.01884799947341283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,0,0.041562666495641075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,0,0.014256000518798828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,0,0.01746133342385292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,0,0.024090667565663654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,0,0.025450666745503742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,0,0.03702933341264725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,0,0.039962666730086006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,0,0.02367466688156128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,0,0.025301332275072735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,0,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,0,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,0,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,0,0.023770667612552643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,0,0.013701333353916803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,0,0.016943999876578648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,0,0.018063999712467194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,0,0.016282666474580765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,0,0.025231999655564625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,0,0.017317333569129307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,0,0.011541333049535751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,0,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,0,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,0,0.012853333105643591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,0,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,0,0.011168000598748526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,0,0.013280000537633896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,0,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,0,0.017322666943073273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,0,0.015967999895413715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,0,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,0,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,0,0.011717333147923151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,0,0.012901333471139273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,0,0.010965333630641302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,0,0.012874666601419449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,0,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,0,0.011855999628702799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,0,0.013125333935022354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,0,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,0,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,0,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,0,0.011648000528415045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,0,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,0,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,0,0.011855999628702799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,0,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,0,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,0,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,0,0.010928000013033548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,0,0.011242666592200598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,0,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,0,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,0,0.01166933278242747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,0,0.010911999891201654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,0,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,0,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,0,0.011578666667143503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,0,0.012597333639860153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,0,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,0,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,0,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,0,0.011066666493813196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,0,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,0,0.01110400011142095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,0,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,0,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,0,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,0,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,0,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,0,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,0,0.01101333275437355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,0,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,0,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,0,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,0,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,0,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,4,1,128,1,float16,float16,0,3.69217586517334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,4,1,128,1,float16,fp8,0,3.6793600718180337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,float16,0,2.33079465230306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,1,128,1,float16,float16,0,1.8891520500183105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,fp8,0,2.4096266428629556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,4,2,128,1,float16,fp8,0,3.815312067667643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,1,128,1,float16,fp8,0,1.9066294034322102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,2,128,1,float16,float16,0,2.0606613159179688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,2,128,1,float16,fp8,0,1.9613760312398274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,4,2,128,1,float16,float16,0,8.991813023885092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,float16,0,1.211349328358968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,float16,0,2.33351993560791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,1,128,1,float16,fp8,0,1.0862293243408203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,fp8,0,1.2714773019154866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,4,4,128,1,float16,fp8,0,2.40940268834432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,2,128,1,float16,fp8,0,1.4525279998779297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,float16,0,0.6750293572743734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,1,128,1,float16,float16,0,0.9942346413930258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,2,128,1,float16,float16,0,1.063920021057129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,float16,0,1.2058080037434895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,4,4,128,1,float16,fp8,0,1.2965599695841472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,fp8,0,0.7075093587239584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,1,128,1,float16,float16,0,0.544543981552124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,1,128,1,float16,fp8,0,0.6403679847717285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,2,128,1,float16,float16,0,0.5922079881032308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,2,128,1,float16,fp8,0,0.5851306517918905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,float16,0,0.6720639864603678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,4,4,128,1,float16,fp8,0,0.7047839959462484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,4,1,128,1,float16,float16,0,2.1447200775146484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,4,1,128,1,float16,fp8,0,2.1406933466593423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,4,2,128,1,float16,float16,0,2.2963627179463706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,float16,0,1.4354933102925618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,fp8,0,1.530373255411784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,4,2,128,1,float16,fp8,0,2.306063969930013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,1,128,1,float16,float16,0,1.1010826428731282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,1,128,1,float16,fp8,0,1.144005298614502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,2,128,1,float16,float16,0,1.1929600238800049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,2,128,1,float16,fp8,0,1.189738670984904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,float16,0,1.4366240501403809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,float16,0,0.7707520325978597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,4,4,128,1,float16,fp8,0,1.5302826563517253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,1,128,1,float16,float16,0,0.6013760169347128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,fp8,0,0.8100533485412598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,1,128,1,float16,fp8,0,0.6129279931386312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,2,128,1,float16,float16,0,0.6588480075200399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,2,128,1,float16,fp8,0,0.6505066553751627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,float16,0,0.7717546621958414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,4,4,128,1,float16,fp8,0,0.8243680000305176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,float16,0,0.4371306498845418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,fp8,0,0.46404798825581867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,1,128,1,float16,float16,0,0.3307680090268453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,1,128,1,float16,fp8,0,0.35258134206136066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,2,128,1,float16,float16,0,0.38127466042836505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,2,128,1,float16,fp8,0,0.3778293530146281
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,float16,0,0.43926934401194256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,4,1,128,1,float16,fp8,0,1.5657119750976562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,4,2,128,1,float16,float16,0,1.662266731262207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,4,4,128,1,float16,fp8,0,0.465557336807251
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,4,1,128,1,float16,float16,0,1.5329866409301758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,float16,0,1.077344020207723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,1,128,1,float16,fp8,0,0.8297013441721598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,fp8,0,1.1556159655253093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,4,2,128,1,float16,fp8,0,2.693205197652181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,1,128,1,float16,float16,0,0.8068053722381592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,2,128,1,float16,float16,0,0.8830026785532633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,2,128,1,float16,fp8,0,0.8759840329488119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,float16,0,1.0834933121999104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,float16,0,0.5797760089238485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,4,4,128,1,float16,fp8,0,1.1522666613260906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,fp8,0,0.6294186512629191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,float16,0,0.5846666495005289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,1,128,1,float16,float16,0,0.4585973421732585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,1,128,1,float16,fp8,0,0.46408534049987793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,2,128,1,float16,float16,0,0.4871893326441447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,2,128,1,float16,fp8,0,0.4915039936701457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,4,4,128,1,float16,fp8,0,0.6315946578979492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,float16,0,0.3376319805781047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,1,128,1,float16,float16,0,0.2595253388086955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,fp8,0,0.36208001772562665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,1,128,1,float16,fp8,0,0.2635626594225566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,2,128,1,float16,float16,0,0.28551467259724933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,2,128,1,float16,fp8,0,0.28362133105595905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,float16,0,0.3327680031458537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,4,4,128,1,float16,fp8,0,0.36665598551432294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,4,1,128,1,float16,float16,0,1.9815093676249187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,4,2,128,1,float16,float16,0,2.259119987487793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,4,1,128,1,float16,fp8,0,2.023205280303955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,float16,0,1.4600480397542317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,fp8,0,1.5605440139770508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,1,128,1,float16,float16,0,1.0125226974487305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,2,128,1,float16,fp8,0,1.1599040031433105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,1,128,1,float16,fp8,0,1.0502986907958984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,2,128,1,float16,float16,0,1.1768426895141602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,fp8,0,1.5677919387817383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,4,2,128,1,float16,fp8,0,5.137749354044597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,4,4,128,1,float16,float16,0,1.457765261332194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,float16,0,0.7554933230082194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,fp8,0,0.8160426616668701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,1,128,1,float16,float16,0,0.5424426794052124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,fp8,0,0.8339200019836426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,1,128,1,float16,fp8,0,0.5608373483022054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,2,128,1,float16,float16,0,0.6225546598434448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,2,128,1,float16,fp8,0,0.6166880130767822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,4,4,128,1,float16,float16,0,0.8008586565653483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,float16,0,0.41527998447418213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,fp8,0,0.4532159964243571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,1,128,1,float16,fp8,0,0.3188106616338094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,1,128,1,float16,float16,0,0.30133867263793945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,2,128,1,float16,float16,0,0.34679468472798664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,2,128,1,float16,fp8,0,0.3499679962793986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,float16,0,0.41835200786590576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,2,128,1,float16,float16,0,0.2031360069910685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,4,4,128,1,float16,fp8,0,0.4489440123240153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,float16,0,0.24848532676696777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,fp8,0,0.2648586630821228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,1,128,1,float16,float16,0,0.1757226586341858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,1,128,1,float16,fp8,0,0.18754132588704428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,2,128,1,float16,fp8,0,0.2121653358141581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,float16,0,0.2494986653327942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,4,4,128,1,float16,fp8,0,0.2685119907061259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,4,1,128,1,float16,fp8,0,1.2146399815877278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,4,2,128,1,float16,float16,0,1.3418879508972168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,fp8,0,1.0260000228881836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,1,128,1,float16,float16,0,0.6208533445994059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,4,1,128,1,float16,float16,0,4.243866602579753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,float16,0,0.9421439965566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,4,2,128,1,float16,fp8,0,1.3374667167663574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,1,128,1,float16,fp8,0,0.6416693528493246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,2,128,1,float16,float16,0,0.7093919912974039
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,2,128,1,float16,fp8,0,0.7003893057505289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,float16,0,0.9402720133463541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,float16,0,0.500928004582723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,1,128,1,float16,fp8,0,0.3510986566543579
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,4,4,128,1,float16,fp8,0,1.1813759803771973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,fp8,0,0.544975996017456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,1,128,1,float16,float16,0,0.33366398016611737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,2,128,1,float16,float16,0,0.3911999861399333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,2,128,1,float16,fp8,0,0.38742931683858234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,float16,0,0.5014880100886027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,4,4,128,1,float16,fp8,0,0.5428906679153442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,float16,0,0.29039466381073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,fp8,0,0.30878400802612305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,1,128,1,float16,float16,0,0.1946613391240438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,1,128,1,float16,fp8,0,0.20313066244125366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,2,128,1,float16,float16,0,0.22546666860580444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,fp8,0,0.1720586617787679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,2,128,1,float16,fp8,0,0.22792534033457437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,float16,0,0.27923200527826947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,2,128,1,float16,float16,0,0.13075199723243713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,4,4,128,1,float16,fp8,0,0.31086933612823486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,float16,0,0.16420800487200418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,1,128,1,float16,float16,0,0.12261866529782613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,1,128,1,float16,fp8,0,0.12940266728401184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,2,128,1,float16,fp8,0,0.1376053293546041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,float16,0,0.1646773318449656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,4,4,128,1,float16,fp8,0,0.17017066478729248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,4,1,128,1,float16,float16,0,1.1407946745554607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,4,1,128,1,float16,fp8,0,2.0782880783081055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,4,2,128,1,float16,float16,0,1.4146186510721843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,float16,0,1.0136160055796306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,4,2,128,1,float16,fp8,0,1.3517492612202961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,fp8,0,1.1218667030334473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,1,128,1,float16,float16,0,0.5866506497065226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,float16,0,1.013535976409912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,4,128,1,float16,fp8,0,1.1178399721781414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,1,128,1,float16,fp8,0,0.6110666592915853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,2,128,1,float16,float16,0,0.7436532974243164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,4,2,128,1,float16,fp8,0,0.9365812937418619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,float16,0,0.5275040070215861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,fp8,0,0.5884533325831095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,1,128,1,float16,float16,0,0.3174293239911397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,fp8,0,0.5750666856765747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,1,128,1,float16,fp8,0,0.3340959946314494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,2,128,1,float16,float16,0,0.39507198333740234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,2,128,1,float16,fp8,0,0.3766666650772095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,4,4,128,1,float16,float16,0,0.534005324045817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,float16,0,0.2909226616223653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,fp8,0,0.31965865691502887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,1,128,1,float16,float16,0,0.18293867508570352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,1,128,1,float16,fp8,0,0.1858933369318644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,2,128,1,float16,float16,0,0.22294400135676065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,fp8,0,0.18358399470647177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,2,128,1,float16,fp8,0,0.2159199913342794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,float16,0,0.2874720096588135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,4,4,128,1,float16,fp8,0,0.3243946631749471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,float16,0,0.16750399271647134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,1,128,1,float16,float16,0,0.10598933696746826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,1,128,1,float16,fp8,0,0.11365333199501038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,2,128,1,float16,float16,0,0.12286399801572163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,2,128,1,float16,fp8,0,0.12541332840919495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,1,128,1,float16,fp8,0,0.07203733424345653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,float16,0,0.166485329469045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,4,4,128,1,float16,fp8,0,0.18314133087793985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,float16,0,0.08309866487979889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,fp8,0,0.093941330909729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,1,128,1,float16,float16,0,0.06609066824118297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,2,128,1,float16,float16,0,0.07160000006357829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,2,128,1,float16,fp8,0,0.0783679982026418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,4,1,128,1,float16,fp8,0,0.7370399634043375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,float16,0,0.08243733147780101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,4,2,128,1,float16,fp8,0,0.8547146320343018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,4,4,128,1,float16,fp8,0,0.09422399600346883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,4,1,128,1,float16,float16,0,0.7470826307932535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,4,2,128,1,float16,float16,0,0.909663995107015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,float16,0,0.716597318649292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,fp8,0,0.7677919864654541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,1,128,1,float16,float16,0,0.40194133917490643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,float16,0,0.7077759901682535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,1,128,1,float16,fp8,0,0.384117325146993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,2,128,1,float16,float16,0,0.4975466728210449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,2,128,1,float16,fp8,0,0.4519146680831909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,4,4,128,1,float16,fp8,0,0.7620266278584799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,float16,0,0.37957334518432617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,fp8,0,0.40380799770355225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,float16,0,0.37566399574279785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,1,128,1,float16,float16,0,0.222762664159139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,1,128,1,float16,fp8,0,0.21363200743993124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,fp8,0,0.22367999951044717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,2,128,1,float16,float16,0,0.2949120004971822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,2,128,1,float16,fp8,0,0.24965333938598633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,4,4,128,1,float16,fp8,0,0.40510932604471844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,float16,0,0.21040532986323038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,1,128,1,float16,float16,0,0.1251573363939921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,float16,0,0.11629866560300191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,1,128,1,float16,fp8,0,0.12139733632405598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,2,128,1,float16,float16,0,0.15798933307329813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,2,128,1,float16,fp8,0,0.14079466462135315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,float16,0,0.21650665998458862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,4,4,128,1,float16,fp8,0,0.2246453364690145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,fp8,0,0.11883733669916789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,1,128,1,float16,float16,0,0.07502399881680806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,1,128,1,float16,fp8,0,0.07521600027879079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,2,128,1,float16,float16,0,0.08390933275222778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,2,128,1,float16,fp8,0,0.0872213343779246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,1,128,1,float16,float16,0,0.05400000015894572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,1,128,1,float16,fp8,0,0.05388799806435903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,float16,0,0.11804266770680745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,4,4,128,1,float16,fp8,0,0.11256532867749532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,float16,0,0.06566399832566579
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,fp8,0,0.07047999898592631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,2,128,1,float16,float16,0,0.05760533114274343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,2,128,1,float16,fp8,0,0.05970133344332377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,float16,0,0.06542933483918507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,4,4,128,1,float16,fp8,0,0.0702453354994456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,4,1,128,1,float16,float16,0,0.7784960269927979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,4,1,128,1,float16,fp8,0,0.7740800380706787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,4,2,128,1,float16,float16,0,1.05512531598409
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,float16,0,0.8320480187733968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,4,2,128,1,float16,fp8,0,0.9368906815846761
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,fp8,0,0.9102880160013834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,1,128,1,float16,float16,0,0.41610666116078693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,1,128,1,float16,fp8,0,0.40435198942820233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,2,128,1,float16,fp8,0,0.48824000358581543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,fp8,0,0.9169812997182211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,2,128,1,float16,float16,0,0.5213813384373983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,4,4,128,1,float16,float16,0,0.8298772970835367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,float16,0,0.42854400475819904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,fp8,0,0.47173865636189777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,1,128,1,float16,float16,0,0.24968532721201578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,float16,0,0.42897601922353107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,1,128,1,float16,fp8,0,0.2208426594734192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,2,128,1,float16,float16,0,0.2836586634318034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,1,128,1,float16,float16,0,0.13395733634630838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,2,128,1,float16,fp8,0,0.26678399244944256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,4,4,128,1,float16,fp8,0,0.46884799003601074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,float16,0,0.2299999992052714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,fp8,0,0.252346674601237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,float16,0,0.12703999876976013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,1,128,1,float16,fp8,0,0.12621333201726279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,2,128,1,float16,float16,0,0.1641973356405894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,1,128,1,float16,float16,0,0.07172800103823344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,2,128,1,float16,fp8,0,0.15075199802716574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,float16,0,0.24544000625610352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,4,4,128,1,float16,fp8,0,0.2550400098164876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,fp8,0,0.13706666231155396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,1,128,1,float16,fp8,0,0.07246933380762736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,fp8,0,0.06425066788991292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,2,128,1,float16,float16,0,0.08475733796755473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,2,128,1,float16,fp8,0,0.08179733157157898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,float16,0,0.1281760036945343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,4,4,128,1,float16,fp8,0,0.13707199692726135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,float16,0,0.059530665477116905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,float16,0,0.06137066582838694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,1,128,1,float16,float16,0,0.042768001556396484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,1,128,1,float16,fp8,0,0.04402133325735728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,2,128,1,float16,float16,0,0.04842666784922282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,2,128,1,float16,fp8,0,0.04894933104515076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,4,4,128,1,float16,fp8,0,0.06381866832574208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,float16,0,0.047040000557899475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,fp8,0,0.050383999943733215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,1,128,1,float16,float16,0,0.03929600119590759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,1,128,1,float16,fp8,0,0.039994666973749794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,2,128,1,float16,float16,0,0.041834667325019836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,2,128,1,float16,fp8,0,0.04358933369318644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,float16,0,0.047482664386431374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,4,4,128,1,float16,fp8,0,0.05020266771316528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,4,1,128,1,float16,float16,0,0.5149813493092855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,float16,0,0.5856906572977701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,4,1,128,1,float16,fp8,0,0.5221706628799438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,4,2,128,1,float16,float16,0,0.6762239933013916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,4,2,128,1,float16,fp8,0,0.6462399959564209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,fp8,0,0.6521973212560018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,1,128,1,float16,float16,0,0.27477333943049115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,1,128,1,float16,fp8,0,0.27668267488479614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,2,128,1,float16,float16,0,0.3689333200454712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,2,128,1,float16,fp8,0,0.3407093286514282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,float16,0,0.5843466520309448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,fp8,0,0.3455359935760498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,1,128,1,float16,float16,0,0.14962133765220642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,1,128,1,float16,fp8,0,0.1536853313446045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,float16,0,0.3051626682281494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,4,4,128,1,float16,fp8,0,0.755407969156901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,2,128,1,float16,float16,0,0.20642666021982828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,2,128,1,float16,fp8,0,0.19043733676274618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,float16,0,0.3064799904823303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,4,4,128,1,float16,fp8,0,0.34432001908620197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,float16,0,0.1649333337942759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,fp8,0,0.18727999925613403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,1,128,1,float16,float16,0,0.08505066235860188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,float16,0,0.08494933446248372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,1,128,1,float16,fp8,0,0.08566932876904805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,2,128,1,float16,float16,0,0.11378666758537292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,2,128,1,float16,fp8,0,0.10874666770299275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,float16,0,0.16642666856447855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,4,4,128,1,float16,fp8,0,0.1868799924850464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,fp8,0,0.09005332986513774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,1,128,1,float16,float16,0,0.05087466537952423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,1,128,1,float16,fp8,0,0.053717335065205894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,2,128,1,float16,float16,0,0.0591893345117569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,2,128,1,float16,fp8,0,0.06241066753864288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,float16,0,0.09032000104586284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,1,128,1,float16,fp8,0,0.03702933341264725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,4,4,128,1,float16,fp8,0,0.09037333726882935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,float16,0,0.04653333127498627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,fp8,0,0.05323199927806854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,1,128,1,float16,float16,0,0.03480533262093862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,2,128,1,float16,float16,0,0.038389332592487335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,2,128,1,float16,fp8,0,0.04152533411979675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,float16,0,0.047040000557899475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,4,4,128,1,float16,fp8,0,0.053690666953722634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,float16,0,0.03805333375930786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,fp8,0,0.04320533573627472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,1,128,1,float16,float16,0,0.03249066571394602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,1,128,1,float16,fp8,0,0.03448000053564707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,2,128,1,float16,float16,0,0.03420799970626831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,2,128,1,float16,fp8,0,0.037392000357309975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,float16,0,0.03862933317820231
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,4,4,128,1,float16,fp8,0,0.04309333364168803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,4,1,128,1,float16,float16,0,0.5740160147349039
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,4,1,128,1,float16,fp8,0,0.5988693237304688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,1,128,1,float16,fp8,0,0.3125759959220886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,2,128,1,float16,float16,0,0.4344853162765503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,1,128,1,float16,float16,0,0.3025546669960022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,fp8,0,0.8713813622792562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,4,2,128,1,float16,float16,0,0.836575984954834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,2,128,1,float16,fp8,0,0.39772268136342365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,float16,0,0.71124267578125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,float16,0,0.37003199259440106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,4,2,128,1,float16,fp8,0,0.7654986381530762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,fp8,0,0.8281546433766683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,1,128,1,float16,fp8,0,0.17269333203633627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,1,128,1,float16,float16,0,0.16336533427238464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,4,4,128,1,float16,float16,0,0.7191200256347656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,fp8,0,0.46938665707906085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,float16,0,0.19612266620000204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,float16,0,0.37112001578013104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,2,128,1,float16,float16,0,0.23167999585469565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,2,128,1,float16,fp8,0,0.21556800603866577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,fp8,0,0.22708799441655478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,4,4,128,1,float16,fp8,0,0.42265065511067706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,1,128,1,float16,float16,0,0.09238400061925252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,2,128,1,float16,float16,0,0.12781866391499838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,1,128,1,float16,fp8,0,0.09734400113423665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,2,128,1,float16,fp8,0,0.12290133039156596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,float16,0,0.19570666551589966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,4,4,128,1,float16,fp8,0,0.22694400946299234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,float16,0,0.10698133707046509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,fp8,0,0.1163146694501241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,1,128,1,float16,float16,0,0.05043200155099233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,1,128,1,float16,fp8,0,0.05422399938106537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,2,128,1,float16,float16,0,0.0611413319905599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,2,128,1,float16,fp8,0,0.06341866652170818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,float16,0,0.10774399836858113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,4,4,128,1,float16,fp8,0,0.12147733569145203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,float16,0,0.04713066418965658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,fp8,0,0.054101333022117615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,1,128,1,float16,float16,0,0.030693332354227703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,1,128,1,float16,fp8,0,0.03348266581694285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,2,128,1,float16,float16,0,0.035599999129772186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,2,128,1,float16,fp8,0,0.03833066672086716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,float16,0,0.04805333415667216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,4,4,128,1,float16,fp8,0,0.05465066432952881
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,float16,0,0.03446933378775915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,fp8,0,0.040261333187421165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,fp8,0,0.04067199925581614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,1,128,1,float16,float16,0,0.02693866689999898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,fp8,0,0.033717334270477295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,1,128,1,float16,fp8,0,0.029818666477998097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,2,128,1,float16,float16,0,0.02926933268706004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,2,128,1,float16,fp8,0,0.03278400003910065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,2,128,1,float16,fp8,0,0.029626667499542236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,4,4,128,1,float16,float16,0,0.03448000053564707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,float16,0,0.029167999823888142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,1,128,1,float16,float16,0,0.025472000241279602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,1,128,1,float16,fp8,0,0.027647999425729115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,2,128,1,float16,float16,0,0.026554666459560394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,float16,0,0.029274667302767437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,4,4,128,1,float16,fp8,0,0.03387200087308884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,4,1,128,1,float16,float16,0,0.41513601938883465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,4,1,128,1,float16,fp8,0,0.454645315806071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,2,128,1,float16,float16,0,0.34623467922210693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,1,128,1,float16,fp8,0,0.24009066820144653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,2,128,1,float16,fp8,0,0.32922132809956867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,1,128,1,float16,float16,0,0.21854400634765625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,fp8,0,0.762938658396403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,4,2,128,1,float16,float16,0,0.66866135597229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,float16,0,0.6378399928410848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,float16,0,0.639792005221049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,4,2,128,1,float16,fp8,0,0.6318293412526449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,4,4,128,1,float16,fp8,0,0.7626773516337076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,float16,0,0.3274986743927002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,fp8,0,0.4028640190760295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,1,128,1,float16,float16,0,0.11973866820335388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,1,128,1,float16,fp8,0,0.1329866647720337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,float16,0,0.1725226640701294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,fp8,0,0.21150400241216025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,2,128,1,float16,fp8,0,0.18096532424290976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,float16,0,0.3246293266614278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,2,128,1,float16,float16,0,0.1934666633605957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,1,128,1,float16,fp8,0,0.07380266487598419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,1,128,1,float16,float16,0,0.06981866558392842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,4,4,128,1,float16,fp8,0,0.3919413487116496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,2,128,1,float16,float16,0,0.10629333058993022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,2,128,1,float16,fp8,0,0.10428266723950703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,float16,0,0.17216000954310098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,4,4,128,1,float16,fp8,0,0.209498663743337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,float16,0,0.0939466655254364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,fp8,0,0.10392533739407857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,float16,0,0.09324799974759419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,1,128,1,float16,float16,0,0.036933332681655884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,1,128,1,float16,fp8,0,0.042277331153551735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,2,128,1,float16,float16,0,0.048245335618654885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,2,128,1,float16,fp8,0,0.05584000051021576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,4,4,128,1,float16,fp8,0,0.1102186640103658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,float16,0,0.0399893323580424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,fp8,0,0.048986668388048805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,1,128,1,float16,float16,0,0.02348266790310542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,float16,0,0.03928533444801966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,1,128,1,float16,fp8,0,0.028181334336598713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,2,128,1,float16,float16,0,0.028218666712443035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,2,128,1,float16,fp8,0,0.03437866767247518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,4,4,128,1,float16,fp8,0,0.04915733138720194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,2,128,1,float16,fp8,0,0.02932800104220708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,float16,0,0.0283146674434344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,fp8,0,0.035802667339642845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,1,128,1,float16,float16,0,0.020655999581019085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,1,128,1,float16,fp8,0,0.02478400121132533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,2,128,1,float16,float16,0,0.023056000471115112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,float16,0,0.028384000062942505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,2,128,1,float16,float16,0,0.02029866725206375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,4,4,128,1,float16,fp8,0,0.035760000348091125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,float16,0,0.0235359991590182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,float16,0,0.023130667706330616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,fp8,0,0.028981332977612812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,1,128,1,float16,float16,0,0.019343999524911244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,1,128,1,float16,fp8,0,0.023018665611743927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,2,128,1,float16,fp8,0,0.025861332813898723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,4,4,128,1,float16,fp8,0,0.028890666862328846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,float16,0,0.019930666933457058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,fp8,0,0.0252960001428922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,1,128,1,float16,float16,0,0.01848000039656957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,1,128,1,float16,fp8,0,0.022511998812357586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,2,128,1,float16,float16,0,0.018794666975736618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,2,128,1,float16,fp8,0,0.024101334313551586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,float16,0,0.020181333025296528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,4,4,128,1,float16,fp8,0,0.024682665864626568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,4,1,128,1,float16,float16,0,0.19317867358525595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,4,1,128,1,float16,fp8,0,0.2174933354059855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,4,2,128,1,float16,float16,0,0.3386880159378052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,float16,0,0.3159839908281962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,fp8,0,0.3702933390935262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,1,128,1,float16,float16,0,0.10803199807802837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,4,2,128,1,float16,fp8,0,0.30483200152715045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,1,128,1,float16,fp8,0,0.12113599975903828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,2,128,1,float16,float16,0,0.18313066164652506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,2,128,1,float16,fp8,0,0.16642666856447855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,float16,0,0.31247466802597046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,float16,0,0.16589867075284323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,4,4,128,1,float16,fp8,0,0.3720746835072835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,fp8,0,0.1997226675351461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,1,128,1,float16,float16,0,0.061808000008265175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,1,128,1,float16,fp8,0,0.06836266815662384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,2,128,1,float16,float16,0,0.09849066535631816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,2,128,1,float16,fp8,0,0.09556266665458679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,float16,0,0.16522666811943054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,4,4,128,1,float16,fp8,0,0.19799466927846274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,float16,0,0.08931733171145122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,fp8,0,0.10106666882832845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,float16,0,0.08789333701133728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,1,128,1,float16,float16,0,0.030661332110563915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,1,128,1,float16,fp8,0,0.0365226666132609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,2,128,1,float16,float16,0,0.04693866769472758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,1,128,1,float16,fp8,0,0.024090667565663654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,2,128,1,float16,fp8,0,0.04775466521581014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,2,128,1,float16,float16,0,0.025589334468046825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,2,128,1,float16,fp8,0,0.030410667260487873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,float16,0,0.03608000030120214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,4,4,128,1,float16,fp8,0,0.09638933340708415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,fp8,0,0.03281066566705704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,float16,0,0.037418665985266365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,fp8,0,0.04621333380540212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,1,128,1,float16,float16,0,0.020794666061798733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,4,4,128,1,float16,fp8,0,0.04580800235271454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,float16,0,0.025621332228183746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,1,128,1,float16,float16,0,0.017792000124851864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,2,128,1,float16,float16,0,0.02012266715367635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,1,128,1,float16,fp8,0,0.020981334149837494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,2,128,1,float16,fp8,0,0.026000000536441803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,float16,0,0.02603200078010559
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,4,4,128,1,float16,fp8,0,0.03293866664171219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,float16,0,0.0206133338312308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,fp8,0,0.026159999271233875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,1,128,1,float16,float16,0,0.01634666696190834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,float16,0,0.017344000438849132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,1,128,1,float16,fp8,0,0.01929066702723503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,1,128,1,float16,float16,0,0.015466666469971338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,2,128,1,float16,float16,0,0.017605333278576534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,2,128,1,float16,fp8,0,0.022181332111358643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,2,128,1,float16,float16,0,0.015568000574906668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,2,128,1,float16,fp8,0,0.01897066707412402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,float16,0,0.02067733307679494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,4,4,128,1,float16,fp8,0,0.026320000489552815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,fp8,0,0.021530665457248688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,1,128,1,float16,fp8,0,0.017925333231687546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,1,128,1,float16,float16,0,0.015344000111023584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,float16,0,0.017173333714405697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,4,4,128,1,float16,fp8,0,0.021066665649414062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,float16,0,0.015423999478419622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,fp8,0,0.019167999426523846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,1,128,1,float16,fp8,0,0.018592000007629395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,2,128,1,float16,float16,0,0.015322666615247726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,2,128,1,float16,fp8,0,0.018837332725524902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,float16,0,0.01598400001724561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,4,4,128,1,float16,fp8,0,0.019178666174411774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,4,1,128,1,float16,float16,0,0.1030560036500295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,4,1,128,1,float16,fp8,0,0.1135040024916331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,4,2,128,1,float16,float16,0,0.1787839929262797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,float16,0,0.16267733772595724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,1,128,1,float16,float16,0,0.061679999033610024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,fp8,0,0.18927999337514242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,4,2,128,1,float16,fp8,0,0.15471466382344565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,float16,0,0.16565866271654764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,1,128,1,float16,fp8,0,0.06332266827424367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,2,128,1,float16,float16,0,0.09573333462079366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,2,128,1,float16,fp8,0,0.08673066894213359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,float16,0,0.08770666519800822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,4,4,128,1,float16,fp8,0,0.18714666366577148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,fp8,0,0.09173333644866943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,1,128,1,float16,float16,0,0.02865600089232127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,1,128,1,float16,fp8,0,0.032442666590213776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,float16,0,0.03522666543722153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,2,128,1,float16,float16,0,0.04518933097521464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,2,128,1,float16,fp8,0,0.03999999910593033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,float16,0,0.0869599978129069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,4,4,128,1,float16,fp8,0,0.0943946639696757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,fp8,0,0.04164800047874451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,1,128,1,float16,float16,0,0.019178666174411774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,1,128,1,float16,fp8,0,0.022656001150608063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,2,128,1,float16,float16,0,0.024298667907714844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,2,128,1,float16,fp8,0,0.027248000105222065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,float16,0,0.0373333344856898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,2,128,1,float16,float16,0,0.018960000326236088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,4,4,128,1,float16,fp8,0,0.04178133110205332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,float16,0,0.024469333390394848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,fp8,0,0.028938665986061096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,fp8,0,0.028677334388097126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,1,128,1,float16,float16,0,0.016309333344300587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,1,128,1,float16,fp8,0,0.019381333142518997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,2,128,1,float16,fp8,0,0.02239999920129776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,4,4,128,1,float16,float16,0,0.024090667565663654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,float16,0,0.01889066646496455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,fp8,0,0.02207999924818675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,1,128,1,float16,float16,0,0.015018666783968607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,float16,0,0.015802666544914246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,1,128,1,float16,fp8,0,0.017349333812793095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,2,128,1,float16,float16,0,0.016149333367745083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,2,128,1,float16,fp8,0,0.01905599981546402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,float16,0,0.01855466639002164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,2,128,1,float16,fp8,0,0.01628799984852473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,4,4,128,1,float16,fp8,0,0.02216000109910965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,fp8,0,0.018325333793958027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,1,128,1,float16,float16,0,0.014069333672523499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,fp8,0,0.015872000406185787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,1,128,1,float16,fp8,0,0.016037333756685257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,2,128,1,float16,float16,0,0.014373333503802618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,float16,0,0.01568000018596649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,4,4,128,1,float16,fp8,0,0.01786133274435997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,float16,0,0.013989333063364029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,1,128,1,float16,float16,0,0.013834666460752487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,1,128,1,float16,fp8,0,0.015909332782030106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,2,128,1,float16,float16,0,0.013749333719412485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,2,128,1,float16,fp8,0,0.01589866727590561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,float16,0,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,4,4,128,1,float16,fp8,0,0.016037333756685257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,2,128,1,float16,fp8,0,0.015615999698638916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,float16,0,0.013621332744757334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,fp8,0,0.015706667055686314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,1,128,1,float16,float16,0,0.01360000049074491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,1,128,1,float16,fp8,0,0.015504000087579092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,2,128,1,float16,float16,0,0.013493333011865616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,float16,0,0.01349866638580958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,4,4,128,1,float16,fp8,0,0.015493333339691162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,4,1,128,1,float16,float16,0,0.05994133154551188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,4,1,128,1,float16,fp8,0,0.0625546673933665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,4,2,128,1,float16,float16,0,0.09559466441472371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,1,128,1,float16,fp8,0,0.03271466741959254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,float16,0,0.08771199981371562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,4,2,128,1,float16,fp8,0,0.08578667044639587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,fp8,0,0.08718933661778767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,1,128,1,float16,float16,0,0.028783999383449554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,2,128,1,float16,float16,0,0.045365333557128906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,fp8,0,0.04119999955097834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,2,128,1,float16,fp8,0,0.039546666045983635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,float16,0,0.08966400225957234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,4,4,128,1,float16,fp8,0,0.09001599748929341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,float16,0,0.036346666514873505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,1,128,1,float16,float16,0,0.019173332800467808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,1,128,1,float16,fp8,0,0.022624000906944275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,2,128,1,float16,float16,0,0.024218666056791942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,2,128,1,float16,fp8,0,0.027221334477265675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,float16,0,0.03642133375008901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,4,4,128,1,float16,fp8,0,0.040922666589419045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,float16,0,0.022384000321229298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,fp8,0,0.024911999702453613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,1,128,1,float16,float16,0,0.01651200031240781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,1,128,1,float16,fp8,0,0.01929066702723503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,2,128,1,float16,float16,0,0.018885333091020584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,fp8,0,0.018298666924238205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,2,128,1,float16,fp8,0,0.02214933435122172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,float16,0,0.02213866760333379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,4,4,128,1,float16,fp8,0,0.024858665963013966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,float16,0,0.01681600014368693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,1,128,1,float16,float16,0,0.015274666249752045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,fp8,0,0.018309333672126133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,1,128,1,float16,fp8,0,0.01756799966096878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,2,128,1,float16,float16,0,0.01613866661985715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,2,128,1,float16,fp8,0,0.01876266673207283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,4,4,128,1,float16,float16,0,0.016895999511082966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,fp8,0,0.015141333142916361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,float16,0,0.014240000396966934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,1,128,1,float16,float16,0,0.014271999398867289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,1,128,1,float16,fp8,0,0.016197333733240765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,2,128,1,float16,float16,0,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,1,128,1,float16,float16,0,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,2,128,1,float16,fp8,0,0.016234666109085083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,float16,0,0.013946666071812311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,4,4,128,1,float16,fp8,0,0.015013333410024643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,float16,0,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,fp8,0,0.012543999900420507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,1,128,1,float16,fp8,0,0.01594666639963786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,2,128,1,float16,float16,0,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,2,128,1,float16,fp8,0,0.015957333147525787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,float16,0,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,4,4,128,1,float16,fp8,0,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,float16,0,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,float16,0,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,fp8,0,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,float16,0,0.011349332829316458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,1,128,1,float16,float16,0,0.013445333888133367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,1,128,1,float16,float16,0,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,1,128,1,float16,fp8,0,0.015589332828919092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,2,128,1,float16,float16,0,0.01360000049074491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,2,128,1,float16,fp8,0,0.015493333339691162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,4,4,128,1,float16,fp8,0,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,fp8,0,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,1,128,1,float16,fp8,0,0.015397333850463232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,2,128,1,float16,float16,0,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,2,128,1,float16,fp8,0,0.015200000256299973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,4,1,128,1,float16,float16,0,0.028650666276613872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,float16,0,0.011616000284751257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,4,4,128,1,float16,fp8,0,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,4,1,128,1,float16,fp8,0,0.03258133431275686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,4,2,128,1,float16,float16,0,0.042709335684776306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,float16,0,0.04320000112056732
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,1,128,1,float16,fp8,0,0.0230880007147789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,4,2,128,1,float16,fp8,0,0.04083733260631561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,fp8,0,0.04709866642951965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,1,128,1,float16,float16,0,0.019354666272799175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,2,128,1,float16,float16,0,0.02548266698916753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,2,128,1,float16,fp8,0,0.026309333741664886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,float16,0,0.043103997906049095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,float16,0,0.025749333202838898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,4,4,128,1,float16,fp8,0,0.04716266691684723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,fp8,0,0.028175999720891316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,1,128,1,float16,float16,0,0.016293333222468693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,1,128,1,float16,fp8,0,0.01911466692884763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,2,128,1,float16,float16,0,0.017130666722853977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,2,128,1,float16,fp8,0,0.018031999468803406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,float16,0,0.025616000096003216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,4,4,128,1,float16,fp8,0,0.028181334336598713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,float16,0,0.016949333250522614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,fp8,0,0.018250666558742523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,1,128,1,float16,float16,0,0.015119999647140503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,1,128,1,float16,fp8,0,0.01741333305835724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,2,128,1,float16,float16,0,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,2,128,1,float16,fp8,0,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,1,128,1,float16,fp8,0,0.015935999651749928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,float16,0,0.016938666502634685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,4,4,128,1,float16,fp8,0,0.01848000039656957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,float16,0,0.014181333283583323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,fp8,0,0.015173333386580149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,fp8,0,0.014970666418472925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,1,128,1,float16,float16,0,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,2,128,1,float16,float16,0,0.012879999975363413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,2,128,1,float16,fp8,0,0.013343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,4,4,128,1,float16,float16,0,0.014416000495354334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,float16,0,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,float16,0,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,fp8,0,0.013157332936922709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,1,128,1,float16,float16,0,0.013744000345468521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,1,128,1,float16,float16,0,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,1,128,1,float16,fp8,0,0.01551466683546702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,1,128,1,float16,fp8,0,0.015669333438078564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,2,128,1,float16,float16,0,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,2,128,1,float16,fp8,0,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,4,4,128,1,float16,fp8,0,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,fp8,0,0.012527999778588613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,fp8,0,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,fp8,0,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,2,128,1,float16,float16,0,0.01179733375708262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,2,128,1,float16,fp8,0,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,2,128,1,float16,float16,0,0.011247999966144562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,4,4,128,1,float16,float16,0,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,float16,0,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,1,128,1,float16,float16,0,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,float16,0,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,1,128,1,float16,fp8,0,0.015274666249752045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,2,128,1,float16,fp8,0,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,float16,0,0.011733333269755045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,2,128,1,float16,float16,0,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,2,128,1,float16,fp8,0,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,4,4,128,1,float16,fp8,0,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,fp8,0,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,1,128,1,float16,float16,0,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,1,128,1,float16,fp8,0,0.015429332852363586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,float16,0,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,4,4,128,1,float16,fp8,0,0.01227733368674914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,4,1,128,1,float16,float16,0,0.020853333175182343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,4,1,128,1,float16,fp8,0,0.02195200075705846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,4,2,128,1,float16,float16,0,0.031957333286603294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,float16,0,0.031898667414983116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,fp8,0,0.03454400102297465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,4,2,128,1,float16,fp8,0,0.03297066688537598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,1,128,1,float16,float16,0,0.014896000425020853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,1,128,1,float16,fp8,0,0.015317333241303762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,2,128,1,float16,float16,0,0.02040533348917961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,2,128,1,float16,fp8,0,0.02160533269246419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,float16,0,0.03196800003449122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,float16,0,0.020474666108687718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,4,4,128,1,float16,fp8,0,0.03469333300987879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,fp8,0,0.022042666872342426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,1,128,1,float16,float16,0,0.012885333349307379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,fp8,0,0.021802666286627453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,1,128,1,float16,fp8,0,0.013552000125249227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,2,128,1,float16,float16,0,0.014479999740918478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,2,128,1,float16,fp8,0,0.015119999647140503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,4,4,128,1,float16,float16,0,0.02080533280968666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,float16,0,0.014405333747466406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,fp8,0,0.015082667271296183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,1,128,1,float16,float16,0,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,1,128,1,float16,fp8,0,0.013066666821638743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,2,128,1,float16,float16,0,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,1,128,1,float16,float16,0,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,1,128,1,float16,fp8,0,0.012949333836634954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,2,128,1,float16,fp8,0,0.013338666409254074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,float16,0,0.014293332894643148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,4,4,128,1,float16,fp8,0,0.01533866673707962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,float16,0,0.012991999586423239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,fp8,0,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,fp8,0,0.01309866706530253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,2,128,1,float16,float16,0,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,2,128,1,float16,fp8,0,0.0129120002190272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,float16,0,0.013050666699806849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,4,4,128,1,float16,fp8,0,0.013232000172138214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,float16,0,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,fp8,0,0.013023999830087027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,1,128,1,float16,float16,0,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,fp8,0,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,1,128,1,float16,float16,0,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,1,128,1,float16,fp8,0,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,2,128,1,float16,float16,0,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,2,128,1,float16,fp8,0,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,4,4,128,1,float16,float16,0,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,float16,0,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,1,128,1,float16,fp8,0,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,fp8,0,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,2,128,1,float16,float16,0,0.011653333902359009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,2,128,1,float16,fp8,0,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,float16,0,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,2,128,1,float16,fp8,0,0.011637333780527115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,4,4,128,1,float16,fp8,0,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,float16,0,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,float16,0,0.011365332951148352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,fp8,0,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,1,128,1,float16,float16,0,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,1,128,1,float16,fp8,0,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,2,128,1,float16,float16,0,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,2,128,1,float16,fp8,0,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,float16,0,0.011978667229413986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,4,4,128,1,float16,fp8,0,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,1,128,1,float16,float16,0,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,1,128,1,float16,fp8,0,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,2,128,1,float16,float16,0,0.011242666592200598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,float16,0,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,4,4,128,1,float16,fp8,0,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,0,0.019797333826621372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,0,0.020879998803138733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,0,0.025589334468046825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,0,0.023957334458827972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,0,0.027301333844661713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,0,0.025744001070658367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,0,0.013893333574136099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,0,0.014842666685581207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,0,0.017450666675964992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,0,0.018805333723624546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,0,0.02420266717672348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,0,0.025706666211287182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,0,0.01340266689658165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,0,0.016949333250522614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,0,0.01817600056529045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,0,0.012752000242471695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,0,0.012949333836634954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,0,0.013616000612576803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,0,0.016794666647911072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,0,0.018085333208243053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,0,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,0,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,0,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,0,0.013114667187134424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,0,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,0,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,0,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,0,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,0,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,0,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,0,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,0,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,0,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,0,0.010837333897749582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,0,0.011546666423479715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,0,0.012842666357755661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,0,0.013045333325862885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,0,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,0,0.012650666137536367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,0,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,0,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,0,0.011157333850860596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,0,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,0,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,0,0.011760000139474869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,0,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,0,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,0,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,0,0.011039999624093374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,0,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,0,0.011626667032639185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,0,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,0,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,0,0.011253333340088526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,0,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,0,0.010863999525705973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,0,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,0,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,0,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,0,0.01138666644692421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,0,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,0,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,0,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,0,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,0,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,0,0.010879999647537867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,0,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,0,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,2,1,128,1,float16,float16,0,1.9607092539469402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,float16,0,1.2103839715321858
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16384,2,1,128,1,float16,fp8,0,1.9793012936909993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,fp8,0,1.1458666324615479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,fp8,0,0.6594026486078898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,1,128,1,float16,float16,0,1.0301706790924072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,1,128,1,float16,fp8,0,1.0473439693450928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,float16,0,1.173456033070882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,2,2,128,1,float16,fp8,0,1.1532213687896729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,float16,0,0.6810773213704427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,1,128,1,float16,float16,0,0.5708853403727213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,1,128,1,float16,fp8,0,0.5852426687876383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,float16,0,0.6614986658096313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,2,2,128,1,float16,fp8,0,0.6468533277511597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,float16,0,0.3988906542460124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,fp8,0,0.40064001083374023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,1,128,1,float16,float16,0,0.3341279824574788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,1,128,1,float16,fp8,0,0.34669868151346844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,float16,0,0.40092798074086505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,fp8,0,0.7239039738972982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,2,2,128,1,float16,fp8,0,0.40033066272735596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,2,1,128,1,float16,float16,0,1.5529866218566895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,12288,2,1,128,1,float16,fp8,0,1.1876479784647624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,float16,0,0.7372533480326334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,1,128,1,float16,float16,0,0.6284586588541666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,1,128,1,float16,fp8,0,0.655509352684021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,float16,0,0.7352213064829508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,2,2,128,1,float16,fp8,0,0.7238933245340983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,float16,0,0.42710399627685547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,fp8,0,0.465887983640035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,1,128,1,float16,float16,0,0.365503986676534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,1,128,1,float16,fp8,0,0.3762880166371663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,float16,0,0.42428267002105713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,2,2,128,1,float16,fp8,0,0.42371733983357746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,float16,0,0.25518399477005005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,fp8,0,0.2498240073521932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,1,128,1,float16,float16,0,0.22438400983810425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,2,1,128,1,float16,fp8,0,0.8766400019327799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,1,128,1,float16,fp8,0,0.2439840038617452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,float16,0,0.2518666585286458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,2,2,128,1,float16,fp8,0,0.2518666585286458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,10240,2,1,128,1,float16,float16,0,0.8478026390075684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,float16,0,0.5671146710713705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,float16,0,0.5533706744511923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,fp8,0,0.5407840013504028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,1,128,1,float16,float16,0,0.4628373384475708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,1,128,1,float16,fp8,0,0.48983999093373615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,2,2,128,1,float16,fp8,0,0.5419840017954508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,float16,0,0.32685865958531696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,float16,0,0.328272004922231
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,fp8,0,0.3248639901479085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,1,128,1,float16,float16,0,0.27532800038655597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,1,128,1,float16,fp8,0,0.2841973304748535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,2,2,128,1,float16,fp8,0,0.32317866881688434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,float16,0,0.1898933251698812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,fp8,0,0.19610132773717245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,1,128,1,float16,float16,0,0.17195733388264975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,1,128,1,float16,fp8,0,0.1849493384361267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,float16,0,0.19419199228286743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,2,2,128,1,float16,fp8,0,0.19501866896947226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,2,1,128,1,float16,float16,0,1.1010826428731282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,8192,2,1,128,1,float16,fp8,0,1.1217652956644695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,float16,0,0.7365226745605469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,float16,0,0.7208853562672933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,fp8,0,0.6889653205871582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,1,128,1,float16,float16,0,0.5784746805826823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,1,128,1,float16,fp8,0,0.6039679845174154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,2,2,128,1,float16,fp8,0,0.6932426293691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,float16,0,0.395087997118632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,fp8,0,0.3906879822413127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,1,128,1,float16,float16,0,0.33026132980982464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,1,128,1,float16,fp8,0,0.34329601128896076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,float16,0,0.4010293483734131
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,2,2,128,1,float16,fp8,0,0.3892213503519694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,float16,0,0.2461599906285604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,fp8,0,0.24250666300455728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,1,128,1,float16,float16,0,0.2010879913965861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,1,128,1,float16,fp8,0,0.20689600706100464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,float16,0,0.247216006120046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,2,2,128,1,float16,fp8,0,0.23768534262975058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,float16,0,0.1339466671148936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,fp8,0,0.139573335647583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,2,1,128,1,float16,float16,0,0.7123733361562093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,1,128,1,float16,float16,0,0.12056000034014384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,1,128,1,float16,fp8,0,0.13025066256523132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,float16,0,0.13245866696039835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,1,128,1,float16,fp8,0,0.3630666732788086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,2,2,128,1,float16,fp8,0,0.13793599605560303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,6144,2,1,128,1,float16,fp8,0,0.6609919865926107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,float16,0,0.4830506642659505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,fp8,0,0.42506134510040283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,1,128,1,float16,float16,0,0.3858826557795207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,float16,0,0.4855039914449056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,2,2,128,1,float16,fp8,0,0.43115198612213135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,float16,0,0.2844640016555786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,fp8,0,0.248799999554952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,1,128,1,float16,float16,0,0.22514132658640543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,1,128,1,float16,float16,0,0.13334932923316956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,1,128,1,float16,fp8,0,0.20989332596460977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,float16,0,0.28196799755096436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,2,2,128,1,float16,fp8,0,0.24941867589950562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,float16,0,0.164682666460673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,fp8,0,0.09902399778366089
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,fp8,0,0.14787733554840088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,1,128,1,float16,fp8,0,0.12669333815574646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,float16,0,0.15946666399637857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,2,2,128,1,float16,fp8,0,0.14601066708564758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,float16,0,0.10237333178520203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,1,128,1,float16,float16,0,0.09507200121879578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,1,128,1,float16,fp8,0,0.0916373332341512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,float16,0,0.10231467088063557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,2,2,128,1,float16,fp8,0,0.10098666946093242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,2,1,128,1,float16,float16,0,0.7014453411102295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,1,128,1,float16,fp8,0,0.3638506730397542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,float16,0,0.5152533451716105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,float16,0,0.27935999631881714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,4096,2,1,128,1,float16,fp8,0,0.6698613166809082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,fp8,0,0.44410133361816406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,1,128,1,float16,fp8,0,0.20698134104410806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,1,128,1,float16,float16,0,0.3765653371810913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,float16,0,0.2803093393643697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,float16,0,0.5058506727218628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,2,2,128,1,float16,fp8,0,0.45269866784413654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,fp8,0,0.2502880096435547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,1,128,1,float16,float16,0,0.21577600638071695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,2,2,128,1,float16,fp8,0,0.25012799104054767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,float16,0,0.1683573325475057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,fp8,0,0.14846400419871011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,1,128,1,float16,float16,0,0.1232373317082723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,1,128,1,float16,fp8,0,0.11853866775830586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,float16,0,0.17029333114624023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,2,2,128,1,float16,fp8,0,0.14916266997655234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,float16,0,0.08290666838486989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,fp8,0,0.0817333310842514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,1,128,1,float16,float16,0,0.07215466598669688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,1,128,1,float16,fp8,0,0.07245866457621257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,float16,0,0.0828000009059906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,float16,0,0.0717386653025945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,2,2,128,1,float16,fp8,0,0.08088000118732452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,float16,0,0.07191999753316243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,fp8,0,0.07297599812348683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,1,128,1,float16,float16,0,0.06638933221499126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,1,128,1,float16,fp8,0,0.06671999891599019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,2,2,128,1,float16,fp8,0,0.07302933434645335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,2,1,128,1,float16,float16,0,0.4506880044937134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,3072,2,1,128,1,float16,fp8,0,0.4451893170674642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,float16,0,0.34690133730570477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,float16,0,0.3445333242416382
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,1,128,1,float16,float16,0,0.2518933415412903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,fp8,0,0.3080586592356364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,1,128,1,float16,float16,0,0.14362133542696634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,1,128,1,float16,fp8,0,0.24676267306009927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,2,2,128,1,float16,fp8,0,0.30669333537419635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,float16,0,0.20125333468119302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,fp8,0,0.17731734116872153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,1,128,1,float16,fp8,0,0.14114133516947427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,float16,0,0.20025600989659628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,2,2,128,1,float16,fp8,0,0.17948265870412192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,float16,0,0.10805333654085796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,fp8,0,0.0981226662794749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,1,128,1,float16,float16,0,0.08171199758847554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,1,128,1,float16,fp8,0,0.08521067102750142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,1,128,1,float16,fp8,0,0.05889600018660227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,float16,0,0.06452799836794536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,float16,0,0.11206400394439697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,2,2,128,1,float16,fp8,0,0.09805333614349365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,float16,0,0.06489066779613495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,fp8,0,0.06629866858323415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,1,128,1,float16,float16,0,0.056874667604764305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,2,2,128,1,float16,fp8,0,0.06565866867701213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,float16,0,0.056688000758488975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,fp8,0,0.05843733251094818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,1,128,1,float16,float16,0,0.05213333169619242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,1,128,1,float16,fp8,0,0.05454400181770325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,float16,0,0.05699199934800466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,2,2,128,1,float16,fp8,0,0.058415999015172325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,2,1,128,1,float16,float16,0,0.481056014696757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,2048,2,1,128,1,float16,fp8,0,0.48232531547546387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,float16,0,0.36772799491882324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,fp8,0,0.34006400903066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,1,128,1,float16,float16,0,0.2622720003128052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,1,128,1,float16,fp8,0,0.26077866554260254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,1,128,1,float16,fp8,0,0.14813866217931113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,float16,0,0.3665173451105754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,2,2,128,1,float16,fp8,0,0.34198931852976483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,float16,0,0.22310932477315268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,fp8,0,0.1906773249308268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,1,128,1,float16,float16,0,0.14692266782124838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,float16,0,0.2251573403676351
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,2,2,128,1,float16,fp8,0,0.19031999508539835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,float16,0,0.1252959966659546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,fp8,0,0.1088746686776479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,1,128,1,float16,float16,0,0.0790880024433136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,1,128,1,float16,fp8,0,0.08164266745249431
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,float16,0,0.125791996717453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,2,2,128,1,float16,fp8,0,0.10559466481208801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,float16,0,0.05850133299827576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,fp8,0,0.05739733576774597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,1,128,1,float16,float16,0,0.04677333434422811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,1,128,1,float16,fp8,0,0.04804266492525736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,float16,0,0.057855998476346336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,2,2,128,1,float16,fp8,0,0.057445332407951355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,float16,0,0.046480000019073486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,fp8,0,0.04828266799449921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,1,128,1,float16,float16,0,0.04137066751718521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,1,128,1,float16,fp8,0,0.04341333111127218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,float16,0,0.046629334489504494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,float16,0,0.04148799926042557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,2,2,128,1,float16,fp8,0,0.04839999973773956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,float16,0,0.04181866844495138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,fp8,0,0.04331199824810028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,1,128,1,float16,float16,0,0.03829866647720337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,1,128,1,float16,fp8,0,0.03944533318281174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,fp8,0,0.2483839988708496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,2,2,128,1,float16,fp8,0,0.04339733223120371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,2,1,128,1,float16,float16,0,0.32848532994588214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1536,2,1,128,1,float16,fp8,0,0.33688000837961835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,float16,0,0.26922667026519775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,1,128,1,float16,float16,0,0.18046400944391885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,1,128,1,float16,fp8,0,0.18493332465489706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,float16,0,0.2773653268814087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,2,2,128,1,float16,fp8,0,0.24830400943756104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,fp8,0,0.14113600055376688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,float16,0,0.15889599919319153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,fp8,0,0.14106667041778564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,1,128,1,float16,float16,0,0.10159466663996379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,1,128,1,float16,fp8,0,0.10569600264231364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,2,2,128,1,float16,float16,0,0.15973866979281107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,float16,0,0.08787199854850769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,float16,0,0.04606399933497111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,fp8,0,0.07400000095367432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,1,128,1,float16,float16,0,0.05630933245023092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,1,128,1,float16,fp8,0,0.062394668658574425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,float16,0,0.08726933598518372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,2,2,128,1,float16,fp8,0,0.07578666508197784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,fp8,0,0.04886933167775472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,1,128,1,float16,float16,0,0.03813866774241129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,1,128,1,float16,fp8,0,0.041296000281969704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,float16,0,0.045797333121299744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,2,2,128,1,float16,fp8,0,0.04861866434415182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,float16,0,0.03810133288304011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,fp8,0,0.04154666761557261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,1,128,1,float16,float16,0,0.03409600009520849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,1,128,1,float16,fp8,0,0.03730666637420654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,float16,0,0.03781333317359289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,2,2,128,1,float16,fp8,0,0.04151466737190882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,float16,0,0.03422400106986364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,fp8,0,0.0379573330283165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,1,128,1,float16,float16,0,0.03249600032965342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,1,128,1,float16,fp8,0,0.035487999518712364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,float16,0,0.033946665624777474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,2,2,128,1,float16,fp8,0,0.03806400050719579
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,2,1,128,1,float16,float16,0,0.37637333075205487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,float16,0,0.3173973361651103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1024,2,1,128,1,float16,fp8,0,0.3942399819691976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,fp8,0,0.2953706582387288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,1,128,1,float16,float16,0,0.20080000162124634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,1,128,1,float16,fp8,0,0.21081066131591797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,float16,0,0.30876266956329346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,float16,0,0.1882879932721456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,float16,0,0.18805332978566489
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,2,2,128,1,float16,fp8,0,0.29367999235788983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,fp8,0,0.16548800468444824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,1,128,1,float16,float16,0,0.1116480032602946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,1,128,1,float16,fp8,0,0.11942399541536967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,2,2,128,1,float16,fp8,0,0.164000004529953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,fp8,0,0.09204266468683879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,float16,0,0.10452266534169515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,fp8,0,0.0897920032342275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,1,128,1,float16,float16,0,0.06122666597366333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,1,128,1,float16,fp8,0,0.06418666740258534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,float16,0,0.04513066510359446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,2,2,128,1,float16,float16,0,0.10455999771753947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,float16,0,0.045791998505592346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,fp8,0,0.04762666424115499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,1,128,1,float16,float16,0,0.03433600068092346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,1,128,1,float16,fp8,0,0.037791999677817024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,2,2,128,1,float16,fp8,0,0.04776533444722494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,float16,0,0.03425599883000056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,fp8,0,0.03881066789229711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,1,128,1,float16,float16,0,0.029301332930723827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,1,128,1,float16,fp8,0,0.03278933217128118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,float16,0,0.03378133227427801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,2,2,128,1,float16,fp8,0,0.03857066730658213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,float16,0,0.028858666618665058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,fp8,0,0.03369600077470144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,1,128,1,float16,float16,0,0.0264533335963885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,1,128,1,float16,fp8,0,0.029893333713213604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,float16,0,0.029120000700155895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,2,2,128,1,float16,fp8,0,0.03342399994532267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,float16,0,0.026159999271233875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,fp8,0,0.029296000798543293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,1,128,1,float16,float16,0,0.02473066747188568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,1,128,1,float16,fp8,0,0.0281333327293396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,float16,0,0.025888000925381977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,2,2,128,1,float16,fp8,0,0.029690665503342945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,2,1,128,1,float16,float16,0,0.291594664255778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,float16,0,0.27448532978693646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,512,2,1,128,1,float16,fp8,0,0.32359999418258667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,fp8,0,0.26102399826049805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,1,128,1,float16,float16,0,0.15555733442306519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,1,128,1,float16,fp8,0,0.17583467562993368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,float16,0,0.2677546739578247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,2,2,128,1,float16,fp8,0,0.26216532786687213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,float16,0,0.16382400194803873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,fp8,0,0.1460533340771993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,1,128,1,float16,float16,0,0.08803733189900716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,1,128,1,float16,fp8,0,0.10130666693051656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,1,128,1,float16,fp8,0,0.0539626677831014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,float16,0,0.15717867016792297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,2,2,128,1,float16,fp8,0,0.1458560029665629
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,float16,0,0.08637332916259766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,fp8,0,0.08007466793060303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,1,128,1,float16,float16,0,0.04789333542188009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,float16,0,0.090229332447052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,2,2,128,1,float16,fp8,0,0.07794133325417836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,float16,0,0.03734933336575826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,fp8,0,0.04224533339341482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,1,128,1,float16,float16,0,0.027349332968393963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,1,128,1,float16,fp8,0,0.033488000432650246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,float16,0,0.03799466788768768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,2,2,128,1,float16,fp8,0,0.042133331298828125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,float16,0,0.027877333263556164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,float16,0,0.023408000667889912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,fp8,0,0.03396799912055334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,1,128,1,float16,float16,0,0.019962667177120846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,1,128,1,float16,float16,0,0.022848000129063923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,1,128,1,float16,fp8,0,0.029045333464940388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,float16,0,0.028410665690898895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,2,2,128,1,float16,fp8,0,0.0340693344672521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,fp8,0,0.02918400118748347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,1,128,1,float16,fp8,0,0.025733334322770435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,float16,0,0.0229066660006841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,2,2,128,1,float16,fp8,0,0.028927999238173168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,fp8,0,0.02497600018978119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,float16,0,0.020010666300853092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,fp8,0,0.025274666647116344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,1,128,1,float16,float16,0,0.018751999984184902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,1,128,1,float16,fp8,0,0.023258666197458904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,2,2,128,1,float16,float16,0,0.02027733375628789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,float16,0,0.018496000518401463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,fp8,0,0.022826666633288067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,1,128,1,float16,float16,0,0.018453333526849747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,1,128,1,float16,fp8,0,0.022682666778564453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,float16,0,0.018496000518401463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,2,2,128,1,float16,fp8,0,0.023061332603295643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,2,1,128,1,float16,float16,0,0.14411733547846475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,float16,0,0.1562879979610443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,256,2,1,128,1,float16,fp8,0,0.16358400384585062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,fp8,0,0.13546133041381836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,1,128,1,float16,float16,0,0.08189333478609721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,1,128,1,float16,fp8,0,0.09265066186587016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,float16,0,0.1593119998772939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,2,2,128,1,float16,fp8,0,0.13343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,float16,0,0.08398399750391643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,fp8,0,0.07098666826883952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,1,128,1,float16,float16,0,0.039264000952243805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,1,128,1,float16,fp8,0,0.04674666623274485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,float16,0,0.08431999882062276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,2,2,128,1,float16,fp8,0,0.07373333474000295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,float16,0,0.03555200000603994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,fp8,0,0.039034667114416756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,fp8,0,0.031210665901501972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,1,128,1,float16,float16,0,0.02422400067249934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,1,128,1,float16,fp8,0,0.02943466603755951
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,float16,0,0.03569599986076355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,2,2,128,1,float16,fp8,0,0.03932266682386398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,float16,0,0.025301332275072735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,1,128,1,float16,float16,0,0.020224000016848247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,float16,0,0.025674665967623394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,1,128,1,float16,fp8,0,0.02548266698916753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,2,2,128,1,float16,fp8,0,0.030810666580994923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,float16,0,0.020047999918460846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,fp8,0,0.02603733291228612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,1,128,1,float16,float16,0,0.017674667139848072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,1,128,1,float16,fp8,0,0.021733333667119343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,float16,0,0.020175999651352566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,2,2,128,1,float16,fp8,0,0.026154667139053345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,float16,0,0.017055999487638474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,fp8,0,0.021759999295075733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,1,128,1,float16,float16,0,0.015882667154073715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,1,128,1,float16,fp8,0,0.01903466631968816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,float16,0,0.0169813334941864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,2,2,128,1,float16,fp8,0,0.021253332495689392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,float16,0,0.015477333217859268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,fp8,0,0.019082666685183842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,1,128,1,float16,float16,0,0.01532799998919169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,1,128,1,float16,fp8,0,0.018800000349680584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,float16,0,0.01544533297419548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,2,2,128,1,float16,fp8,0,0.019130667050679524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,float16,0,0.015178666760524115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,fp8,0,0.018629333625237148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,1,128,1,float16,float16,0,0.014858666807413101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,1,128,1,float16,fp8,0,0.018458666900793713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,float16,0,0.014938666174809137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,2,2,128,1,float16,fp8,0,0.018650667121013004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,2,1,128,1,float16,float16,0,0.07961066563924153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,float16,0,0.08291733264923096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,128,2,1,128,1,float16,fp8,0,0.08362133304278056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,float16,0,0.08306666711966197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,fp8,0,0.06468266745408376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,2,128,1,float16,fp8,0,0.06297066807746887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,1,128,1,float16,float16,0,0.03957866628964742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,2,1,128,1,float16,fp8,0,0.04011200120051702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,float16,0,0.034101332227389015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,fp8,0,0.03515733281771342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,1,128,1,float16,float16,0,0.0232640008131663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,1,128,1,float16,fp8,0,0.026554666459560394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,float16,0,0.03366933266321818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,2,2,128,1,float16,fp8,0,0.0351946676770846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,float16,0,0.02383466561635335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,fp8,0,0.026880001028378803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,1,128,1,float16,float16,0,0.01868266612291336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,1,128,1,float16,fp8,0,0.022122666239738464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,float16,0,0.02364266663789749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,2,2,128,1,float16,fp8,0,0.02661866694688797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,fp8,0,0.021840001145998638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,float16,0,0.0183999997874101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,fp8,0,0.021920000513394673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,1,128,1,float16,float16,0,0.01609066625436147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,1,128,1,float16,fp8,0,0.01903466631968816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,2,2,128,1,float16,float16,0,0.018858666221300762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,float16,0,0.01586666703224182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,fp8,0,0.018112000077962875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,1,128,1,float16,float16,0,0.014384000251690546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,1,128,1,float16,fp8,0,0.01741333305835724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,float16,0,0.015541333705186844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,float16,0,0.014058666924635569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,2,2,128,1,float16,fp8,0,0.018042666216691334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,float16,0,0.014085333794355392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,fp8,0,0.01605333387851715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,1,128,1,float16,float16,0,0.013893333574136099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,1,128,1,float16,fp8,0,0.015669333438078564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,2,2,128,1,float16,fp8,0,0.01602666700879733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,float16,0,0.013669333110253016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,fp8,0,0.015344000111023584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,1,128,1,float16,float16,0,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,1,128,1,float16,fp8,0,0.015552000453074774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,float16,0,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,2,2,128,1,float16,fp8,0,0.015487999965747198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,fp8,0,0.015557333827018738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,float16,0,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,fp8,0,0.015418666104475657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,1,128,1,float16,float16,0,0.013552000125249227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,1,128,1,float16,fp8,0,0.015317333241303762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,2,2,128,1,float16,float16,0,0.013354666531085968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,2,1,128,1,float16,float16,0,0.036890665690104164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,float16,0,0.03523733218510946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,64,2,1,128,1,float16,fp8,0,0.03976000100374222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,fp8,0,0.03450666616360346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,1,128,1,float16,float16,0,0.02319466571013133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,1,128,1,float16,fp8,0,0.026506667335828144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,float16,0,0.03605866680542628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,float16,0,0.02179733415444692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,1,128,1,float16,fp8,0,0.022250667214393616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,2,2,128,1,float16,fp8,0,0.034661332766215004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,fp8,0,0.02317333221435547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,1,128,1,float16,float16,0,0.018725333114465077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,float16,0,0.02176533391078313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,2,2,128,1,float16,fp8,0,0.02266666789849599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,float16,0,0.01655999943614006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,fp8,0,0.018298666924238205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,1,128,1,float16,float16,0,0.01590399940808614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,1,128,1,float16,fp8,0,0.018885333091020584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,float16,0,0.016629333297411602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,2,2,128,1,float16,fp8,0,0.018138666947682697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,float16,0,0.01394133393963178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,fp8,0,0.014890667051076889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,1,128,1,float16,float16,0,0.0143306665122509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,1,128,1,float16,fp8,0,0.01573866605758667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,float16,0,0.014352000008026758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,2,2,128,1,float16,fp8,0,0.015050667027632395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,float16,0,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,fp8,0,0.012709333250919977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,1,128,1,float16,float16,0,0.013951999445756277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,1,128,1,float16,fp8,0,0.01552533358335495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,float16,0,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,float16,0,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,2,2,128,1,float16,fp8,0,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,float16,0,0.01146666705608368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,float16,0,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,fp8,0,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,1,128,1,float16,float16,0,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,1,128,1,float16,fp8,0,0.01532799998919169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,2,2,128,1,float16,fp8,0,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,fp8,0,0.011850666254758835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,1,128,1,float16,float16,0,0.01328533391157786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,1,128,1,float16,fp8,0,0.015392000476519266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,float16,0,0.011557333171367645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,2,2,128,1,float16,fp8,0,0.011936000237862269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,float16,0,0.011312000453472137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,fp8,0,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,1,128,1,float16,float16,0,0.013242666920026144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,1,128,1,float16,fp8,0,0.01551466683546702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,float16,0,0.011621333658695221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,2,2,128,1,float16,fp8,0,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,2,1,128,1,float16,float16,0,0.024847999215126038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,float16,0,0.02526933451493581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,32,2,1,128,1,float16,fp8,0,0.02613866577545802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,fp8,0,0.02622399975856145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,fp8,0,0.026367999613285065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,1,128,1,float16,float16,0,0.016879999389251072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,1,128,1,float16,fp8,0,0.018650667121013004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,2,2,128,1,float16,float16,0,0.025279998779296875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,float16,0,0.017018667111794155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,fp8,0,0.018090666582187016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,fp8,0,0.018122666825850803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,fp8,0,0.014773332824309668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,1,128,1,float16,float16,0,0.014250667144854864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,1,128,1,float16,fp8,0,0.014933332800865173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,2,2,128,1,float16,float16,0,0.016965333372354507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,float16,0,0.0144213338692983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,1,128,1,float16,float16,0,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,1,128,1,float16,fp8,0,0.013482666263977686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,float16,0,0.013909333695967993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,2,2,128,1,float16,fp8,0,0.01471466695268949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,float16,0,0.012768000364303589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,fp8,0,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,fp8,0,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,1,128,1,float16,float16,0,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,1,128,1,float16,fp8,0,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,float16,0,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,2,2,128,1,float16,fp8,0,0.013343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,float16,0,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,fp8,0,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,1,128,1,float16,float16,0,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,1,128,1,float16,fp8,0,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,float16,0,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,2,2,128,1,float16,fp8,0,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,float16,0,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,float16,0,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,1,128,1,float16,float16,0,0.011338666081428528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,1,128,1,float16,fp8,0,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,float16,0,0.011829332758982977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,2,2,128,1,float16,fp8,0,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,fp8,0,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,1,128,1,float16,float16,0,0.011301333705584208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,1,128,1,float16,float16,0,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,1,128,1,float16,fp8,0,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,1,128,1,float16,fp8,0,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,float16,0,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,2,2,128,1,float16,fp8,0,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,float16,0,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,fp8,0,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,float16,0,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,2,2,128,1,float16,fp8,0,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,2,1,128,1,float16,float16,0,0.02063999945918719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,float16,0,0.02045866722861926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,16,2,1,128,1,float16,fp8,0,0.0222080002228419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,fp8,0,0.021754667162895203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,1,128,1,float16,float16,0,0.014783999572197596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,1,128,1,float16,fp8,0,0.015413332730531693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,float16,0,0.020666666328907013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,2,2,128,1,float16,fp8,0,0.02179199953873952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,float16,0,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,float16,0,0.014490666488806406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,fp8,0,0.015290666371583939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,1,128,1,float16,float16,0,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,1,128,1,float16,fp8,0,0.013765333841244379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,2,2,128,1,float16,fp8,0,0.015189333508412043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,float16,0,0.012714666624863943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,fp8,0,0.013514666507641474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,1,128,1,float16,float16,0,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,1,128,1,float16,fp8,0,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,float16,0,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,2,2,128,1,float16,fp8,0,0.013461332768201828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,float16,0,0.012549333274364471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,fp8,0,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,1,128,1,float16,float16,0,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,1,128,1,float16,fp8,0,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,float16,0,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,2,2,128,1,float16,fp8,0,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,float16,0,0.011663999408483505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,float16,0,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,fp8,0,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,fp8,0,0.012373333175977072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,1,128,1,float16,float16,0,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,1,128,1,float16,fp8,0,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,fp8,0,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,float16,0,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,2,2,128,1,float16,fp8,0,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,1,128,1,float16,float16,0,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,1,128,1,float16,fp8,0,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,2,2,128,1,float16,float16,0,0.011770666887362799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,float16,0,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,fp8,0,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,1,128,1,float16,float16,0,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,1,128,1,float16,fp8,0,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,1,128,1,float16,fp8,0,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,float16,0,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,2,2,128,1,float16,fp8,0,0.011706666400035223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,float16,0,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,fp8,0,0.011760000139474869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,1,128,1,float16,float16,0,0.011077333241701126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,float16,0,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,0,0.018794666975736618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,2,2,128,1,float16,fp8,0,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,0,0.01971199984351794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,0,0.01743999992807706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,0,0.02089600016673406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,0,0.013754667093356451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,0,0.014831999937693277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,0,0.017258666455745697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,0,0.01889066646496455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,0,0.012874666601419449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,0,0.013690666606028875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,0,0.013631999492645264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,0,0.012736000120639801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,0,0.013712000101804733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,0,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,0,0.011760000139474869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,0,0.013034666577974955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,0,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,0,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,0,0.011653333902359009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,0,0.012954667210578918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,0,0.013104000439246496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,0,0.019904000063737232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,0,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,0,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,0,0.011994666109482447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,0,0.011589333415031433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,0,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,0,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,0,0.011157333850860596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,0,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,0,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,0,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,0,0.011178666104873022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,0,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,0,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,0,0.010816000401973724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,0,0.011301333705584208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,0,0.012736000120639801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,0,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,0,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,0,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,0,0.0107893335322539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,0,0.011178666104873022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,0,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,0,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,0,0.011994666109482447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,float16,0,0.6195840040842692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,0,0.010890666395425797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,0,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,0,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,0,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,fp8,0,0.642250657081604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,float16,0,0.6209493478139242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16384,1,1,128,1,float16,fp8,0,0.6498186588287354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,float16,0,0.3829760154088338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,fp8,0,0.40090131759643555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,float16,0,0.38473065694173175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16384,1,1,128,1,float16,fp8,0,0.3992160161336263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,float16,0,0.22453866402308145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,fp8,0,0.243450661500295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,float16,0,0.22516266504923502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,float16,0,0.41473066806793213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16384,1,1,128,1,float16,fp8,0,0.24044267336527506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,float16,0,0.4155306816101074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,fp8,0,0.38532265027364093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,12288,1,1,128,1,float16,fp8,0,0.3829439878463745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,float16,0,0.1768640081087748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,float16,0,0.2534613410631816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,float16,0,0.1767573356628418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,fp8,0,0.22942399978637695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,float16,0,0.2545013427734375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,fp8,0,0.29764266808827716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,12288,1,1,128,1,float16,fp8,0,0.22750399510065714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,fp8,0,0.164901336034139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,12288,1,1,128,1,float16,fp8,0,0.16470932960510254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,float16,0,0.3179253339767456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,float16,0,0.3174346685409546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,10240,1,1,128,1,float16,fp8,0,0.29579732815424603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,float16,0,0.19031467040379843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,fp8,0,0.17824000120162964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,float16,0,0.19241599241892496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,10240,1,1,128,1,float16,fp8,0,0.17731734116872153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,float16,0,0.1486240029335022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,fp8,0,0.14110933740933737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,float16,0,0.1488533318042755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,10240,1,1,128,1,float16,fp8,0,0.14043733477592468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,float16,0,0.3829546769460042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,fp8,0,0.3647093375523885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,float16,0,0.38339734077453613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,8192,1,1,128,1,float16,fp8,0,0.3643253246943156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,float16,0,0.23412799835205078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,fp8,0,0.22006400426228842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,float16,0,0.2347466746966044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,8192,1,1,128,1,float16,fp8,0,0.21943465868631998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,float16,0,0.13286933302879333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,fp8,0,0.126202662785848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,float16,0,0.1306933363278707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,8192,1,1,128,1,float16,fp8,0,0.12636267145474753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,float16,0,0.1197653313477834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,fp8,0,0.11665067076683044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,float16,0,0.12087999780972798
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,8192,1,1,128,1,float16,fp8,0,0.11682132879892985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,float16,0,0.25647467374801636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,fp8,0,0.246671994527181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,float16,0,0.2559573252995809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,6144,1,1,128,1,float16,fp8,0,0.24497600396474203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,float16,0,0.1530933380126953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,fp8,0,0.14458133776982626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,float16,0,0.15321066975593567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,6144,1,1,128,1,float16,fp8,0,0.14462400476137796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,float16,0,0.10087999701499939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,fp8,0,0.09935999910036723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,float16,0,0.10002133250236511
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,6144,1,1,128,1,float16,fp8,0,0.09872000416119893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,float16,0,0.09267733494440715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,fp8,0,0.09114133318265279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,float16,0,0.0935093363126119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,6144,1,1,128,1,float16,fp8,0,0.09118933478991191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,float16,0,0.2518879969914754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,fp8,0,0.24875734249750772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,float16,0,0.2523146669069926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,4096,1,1,128,1,float16,fp8,0,0.24702932437260947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,float16,0,0.14969600240389505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,fp8,0,0.14426666498184204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,float16,0,0.14869866768519083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,4096,1,1,128,1,float16,fp8,0,0.14381333192189535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,float16,0,0.08070399860541026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,fp8,0,0.0812853326400121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,float16,0,0.07855999966462453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,float16,0,0.06622399886449178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,4096,1,1,128,1,float16,fp8,0,0.08029866715272267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,float16,0,0.07035199801127116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,fp8,0,0.07161066432793935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,float16,0,0.07060266534487407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,4096,1,1,128,1,float16,fp8,0,0.07157333195209503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,fp8,0,0.06644266843795776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,float16,0,0.06560533245404561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,4096,1,1,128,1,float16,fp8,0,0.0661599983771642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,fp8,0,0.1753013332684835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,fp8,0,0.10026133060455322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,float16,0,0.1750719944636027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,float16,0,0.17577600479125977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,fp8,0,0.06530133386452992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,3072,1,1,128,1,float16,fp8,0,0.1752799948056539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,float16,0,0.10122133294741313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,float16,0,0.05590933561325073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,fp8,0,0.10035199920336406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,3072,1,1,128,1,float16,float16,0,0.10104533036549886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,float16,0,0.06234133243560791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,float16,0,0.0629120022058487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,3072,1,1,128,1,float16,fp8,0,0.06530666848023732
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,fp8,0,0.0584746648867925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,float16,0,0.05310399830341339
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,float16,0,0.055919999877611794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,3072,1,1,128,1,float16,fp8,0,0.05880000193913778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,float16,0,0.05320000151793162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,fp8,0,0.05459199845790863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,3072,1,1,128,1,float16,fp8,0,0.05494933327039083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,float16,0,0.1808799902598063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,fp8,0,0.18820265928904215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,float16,0,0.18414932489395142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,2048,1,1,128,1,float16,fp8,0,0.18690667549769083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,float16,0,0.1065013309319814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,fp8,0,0.10667733351389568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,fp8,0,0.10613866647084554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,float16,0,0.04517866671085358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,2048,1,1,128,1,float16,float16,0,0.10459199547767639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,float16,0,0.05310933291912079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,fp8,0,0.05629866818586985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,float16,0,0.05366399884223938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,fp8,0,0.0433599998553594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,2048,1,1,128,1,float16,fp8,0,0.05637866755326589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,fp8,0,0.04751466711362203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,fp8,0,0.0432640016078949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,float16,0,0.0454773356517156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,2048,1,1,128,1,float16,fp8,0,0.04752000172932943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,fp8,0,0.039221333960692085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,float16,0,0.04155199974775314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,2048,1,1,128,1,float16,float16,0,0.041450666884581246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,float16,0,0.038378665844599404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,float16,0,0.13147200147310892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,fp8,0,0.039861333866914116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,2048,1,1,128,1,float16,float16,0,0.03835733234882355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,float16,0,0.13210666179656982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,fp8,0,0.13859732945760092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1536,1,1,128,1,float16,fp8,0,0.13711999853452048
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,float16,0,0.07649600009123485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,fp8,0,0.0751093327999115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,fp8,0,0.048437332113583885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,float16,0,0.07592533528804779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1536,1,1,128,1,float16,fp8,0,0.07226666808128357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,float16,0,0.043663998444875084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,fp8,0,0.048672000567118325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1536,1,1,128,1,float16,float16,0,0.04334400097529093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,float16,0,0.03735466549793879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,fp8,0,0.04137066751718521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,float16,0,0.03741333385308584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1536,1,1,128,1,float16,fp8,0,0.041519999504089355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,float16,0,0.03421866645415624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,fp8,0,0.03783999880154928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,float16,0,0.03400533397992452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1536,1,1,128,1,float16,fp8,0,0.03782933453718821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,float16,0,0.03181333343187968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,fp8,0,0.034586665530999504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,float16,0,0.032069332897663116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1536,1,1,128,1,float16,fp8,0,0.03480000048875809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,float16,0,0.15016000469525656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,fp8,0,0.16081066926320395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,float16,0,0.14875200390815735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1024,1,1,128,1,float16,fp8,0,0.16033599774042764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,float16,0,0.08680533369382222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,fp8,0,0.08892266949017842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,float16,0,0.08569066723187764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1024,1,1,128,1,float16,fp8,0,0.09108266234397888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,float16,0,0.041237334410349526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,fp8,0,0.04668800036112467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,float16,0,0.04213866591453552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1024,1,1,128,1,float16,fp8,0,0.04670399924119314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,fp8,0,0.03316266586383184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,float16,0,0.03297066688537598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,fp8,0,0.03800000001986822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,float16,0,0.03289066751797994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1024,1,1,128,1,float16,fp8,0,0.03817066550254822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,float16,0,0.02890666574239731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,float16,0,0.028901333610216778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1024,1,1,128,1,float16,fp8,0,0.033546666304270424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,float16,0,0.025941332181294758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,float16,0,0.024442667762438457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,fp8,0,0.02956799914439519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,float16,0,0.027786667148272198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1024,1,1,128,1,float16,fp8,0,0.02959999938805898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,float16,0,0.024490666886170704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,fp8,0,0.02809600035349528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1024,1,1,128,1,float16,fp8,0,0.027717334528764088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,float16,0,0.1260640025138855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,fp8,0,0.14166399836540222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,float16,0,0.12618133425712585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,512,1,1,128,1,float16,fp8,0,0.14174399773279825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,float16,0,0.07163733243942261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,fp8,0,0.07786666850248973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,float16,0,0.07229333122571309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,512,1,1,128,1,float16,fp8,0,0.07721066474914551
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,float16,0,0.033600000043710075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,fp8,0,0.04124800115823746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,float16,0,0.03453333427508672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,float16,0,0.02271466702222824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,512,1,1,128,1,float16,fp8,0,0.04084266722202301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,float16,0,0.026591998835404713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,fp8,0,0.03357866654793421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,float16,0,0.02696000039577484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,512,1,1,128,1,float16,fp8,0,0.03297066688537598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,fp8,0,0.024986666937669117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,fp8,0,0.028832000990708668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,fp8,0,0.024608001112937927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,float16,0,0.02269333352645238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,512,1,1,128,1,float16,fp8,0,0.028581333657105763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,float16,0,0.018581333259741466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,float16,0,0.020256000260512035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,512,1,1,128,1,float16,float16,0,0.020058666666348774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,float16,0,0.018511999398469925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,fp8,0,0.023183998962243397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,512,1,1,128,1,float16,fp8,0,0.02293333411216736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,float16,0,0.017893332988023758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,fp8,0,0.02252800017595291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,float16,0,0.018122666825850803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,512,1,1,128,1,float16,fp8,0,0.022672000030676525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,float16,0,0.0656160016854604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,fp8,0,0.07136000196139018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,float16,0,0.06607999900976817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,float16,0,0.031104000906149547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,fp8,0,0.03833066672086716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,256,1,1,128,1,float16,fp8,0,0.06870933373769124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,fp8,0,0.029685333371162415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,float16,0,0.023754666248957317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,fp8,0,0.03792533278465271
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,float16,0,0.020106667031844456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,float16,0,0.01987733319401741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,fp8,0,0.026026666164398193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,fp8,0,0.021061333517233532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,256,1,1,128,1,float16,float16,0,0.03136000037193298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,float16,0,0.024090667565663654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,256,1,1,128,1,float16,fp8,0,0.030224000414212544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,256,1,1,128,1,float16,fp8,0,0.02565866708755493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,float16,0,0.017162666966517765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,float16,0,0.017210666090250015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,float16,0,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,256,1,1,128,1,float16,fp8,0,0.02124800036350886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,float16,0,0.015610666324694952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,fp8,0,0.01918399954835574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,float16,0,0.015829333414634068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,256,1,1,128,1,float16,fp8,0,0.019082666685183842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,fp8,0,0.01874133323629697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,float16,0,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,256,1,1,128,1,float16,fp8,0,0.018496000518401463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,float16,0,0.014943999548753103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,fp8,0,0.01842133328318596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,float16,0,0.015146666516860327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,float16,0,0.030000001192092896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,256,1,1,128,1,float16,fp8,0,0.018719999740521114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,float16,0,0.03216533362865448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,fp8,0,0.0339626669883728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,128,1,1,128,1,float16,fp8,0,0.03426666557788849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,float16,0,0.022495999932289124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,fp8,0,0.026181332767009735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,float16,0,0.02266666789849599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,128,1,1,128,1,float16,fp8,0,0.026202666262785595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,fp8,0,0.01781333362062772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,float16,0,0.01841066653529803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,fp8,0,0.0216799999276797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,float16,0,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,float16,0,0.018618666877349217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,128,1,1,128,1,float16,fp8,0,0.02182399978240331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,fp8,0,0.015802666544914246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,float16,0,0.015696000307798386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,float16,0,0.015498666713635126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,128,1,1,128,1,float16,fp8,0,0.018170667191346485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,fp8,0,0.01617066686352094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,float16,0,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,128,1,1,128,1,float16,float16,0,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,float16,0,0.013738666971524557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,fp8,0,0.015360000232855478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,fp8,0,0.01553600033124288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,fp8,0,0.015487999965747198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,float16,0,0.013514666507641474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,128,1,1,128,1,float16,fp8,0,0.015274666249752045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,fp8,0,0.01534933348496755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,128,1,1,128,1,float16,float16,0,0.01350933313369751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,float16,0,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,float16,0,0.01340266689658165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,128,1,1,128,1,float16,fp8,0,0.015365333606799444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,float16,0,0.02092266579469045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,fp8,0,0.018170667191346485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,fp8,0,0.02237333357334137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,float16,0,0.02092266579469045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,float16,0,0.016773333152135212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,64,1,1,128,1,float16,fp8,0,0.022272000710169475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,float16,0,0.016634666671355564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,64,1,1,128,1,float16,fp8,0,0.018031999468803406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,float16,0,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,fp8,0,0.014949332922697067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,float16,0,0.01403733342885971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,64,1,1,128,1,float16,fp8,0,0.014997333288192749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,float16,0,0.012719999998807907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,fp8,0,0.012837332983811697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,fp8,0,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,float16,0,0.012256000190973282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,64,1,1,128,1,float16,fp8,0,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,fp8,0,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,float16,0,0.012229333321253458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,fp8,0,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,64,1,1,128,1,float16,float16,0,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,float16,0,0.011578666667143503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,float16,0,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,fp8,0,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,64,1,1,128,1,float16,fp8,0,0.011861333002646765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,float16,0,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,fp8,0,0.011920000116030375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,fp8,0,0.011695999652147293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,64,1,1,128,1,float16,float16,0,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,float16,0,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,fp8,0,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,64,1,1,128,1,float16,float16,0,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,float16,0,0.016906666258970898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,fp8,0,0.018079999834299088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,float16,0,0.016821333517630894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,float16,0,0.014250667144854864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,fp8,0,0.014991999914248785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,32,1,1,128,1,float16,fp8,0,0.018191999445358913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,fp8,0,0.015130666395028433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,32,1,1,128,1,float16,float16,0,0.014170666535695394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,fp8,0,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,float16,0,0.01301866645614306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,fp8,0,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,float16,0,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,32,1,1,128,1,float16,fp8,0,0.013167999684810638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,float16,0,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,float16,0,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,32,1,1,128,1,float16,fp8,0,0.012304000556468964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,fp8,0,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,float16,0,0.011557333171367645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,fp8,0,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,float16,0,0.011621333658695221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,32,1,1,128,1,float16,fp8,0,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,float16,0,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,float16,0,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,float16,0,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,32,1,1,128,1,float16,fp8,0,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,float16,0,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,fp8,0,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,32,1,1,128,1,float16,fp8,0,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,float16,0,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,fp8,0,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,float16,0,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,32,1,1,128,1,float16,fp8,0,0.011802667131026586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,float16,0,0.014922666052977243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,fp8,0,0.015370666980743408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,float16,0,0.014639999717473984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,16,1,1,128,1,float16,fp8,0,0.015370666980743408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,float16,0,0.013104000439246496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,fp8,0,0.013637332866589228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,float16,0,0.012938667088747025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,16,1,1,128,1,float16,fp8,0,0.013776000589132309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,float16,0,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,float16,0,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,fp8,0,0.012736000120639801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,float16,0,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,16,1,1,128,1,float16,fp8,0,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,float16,0,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,float16,0,0.011695999652147293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,fp8,0,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,fp8,0,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,16,1,1,128,1,float16,fp8,0,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,float16,0,0.011861333002646765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,fp8,0,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,fp8,0,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,float16,0,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,16,1,1,128,1,float16,fp8,0,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,float16,0,0.011215999722480774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,float16,0,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,16,1,1,128,1,float16,fp8,0,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,float16,0,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,16,1,1,128,1,float16,fp8,0,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,float16,0,0.010863999525705973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,fp8,0,0.011829332758982977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,float16,0,0.010992000500361124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,0,0.01422400027513504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,16,1,1,128,1,float16,fp8,0,0.01166933278242747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,0,0.013946666071812311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,0,0.014826666563749313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,0,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,0,0.014965333044528961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,0,0.013466666142145792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,0,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,0,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,0,0.013397333522637686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,0,0.011663999408483505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,0,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,0,0.01163200040658315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,0,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,0,0.010911999891201654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,0,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,0,0.010693332801262537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,0,0.011247999966144562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,0,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,0,0.01101333275437355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,0,0.011786667009194693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,0,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,0,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,0,0.010970667004585266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,0,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,0,0.011477333803971609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,0,0.010784000158309937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,0,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,0,0.010847999403874079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,0,0.010965333630641302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,0,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,0,0.010842667271693548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,context_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,0,0.011823999385039011
