framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,31,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,31,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,63,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,63,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,127,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,127,0.014618666221698126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,255,0.013786666095256805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,255,0.015194666882356008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,511,0.015717333803574245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,511,0.017162666966517765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,1023,0.020096000283956528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,1,0.012896000097195307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,1023,0.022495999932289124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,3,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,3,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,2047,0.029653333127498627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,2047,0.03178666780392329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,4095,0.0495413343111674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,4095,0.05399466554323832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,15,0.012719999998807907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,fp8,7,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,15,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,1,0.012725333372751871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,3,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,3,0.012847999731699625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,7,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,7,0.012661332885424295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,15,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,1,128,1,float16,float16,7,0.012373333175977072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,15,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,31,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,31,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,63,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,63,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,127,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,127,0.014266667266686758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,255,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,255,0.015135999768972397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,511,0.01575999955336253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,511,0.017893332988023758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,1023,0.020938667158285778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,1023,0.023584000766277313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,2047,0.03676799933115641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,2047,0.03335466732581457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,float16,4095,0.05947199960549673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,2,128,1,float16,fp8,4095,0.057664001981417336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,7,0.013904000322024027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,1,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,3,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,3,0.014021333307027817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,7,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,15,0.012800000607967377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,15,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,31,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,31,0.013807999591032663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,63,0.012618667135636011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,63,0.013701333353916803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,127,0.01333333303531011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,127,0.014815999815861383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,255,0.014991999914248785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,255,0.01569066693385442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,511,0.017114666601022083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,511,0.01814933369557063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,1023,0.03513066718975703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,1023,0.024031999210516613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,2047,0.05650666852792104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,2047,0.0406986673672994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,3,0.016522667060295742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,3,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,7,0.01659199967980385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,float16,4095,0.09909333785374959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,4,128,1,float16,fp8,4095,0.06400533517201741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1,0.016442666451136272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,1,0.017456000049908955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,7,0.01727466657757759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,15,0.016538667182127636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,15,0.01717866708834966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,31,0.016810666769742966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,31,0.017445333302021027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,63,0.016554666062196095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,63,0.01716800034046173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,511,0.03781333317359289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,127,0.016693333784739178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,127,0.02059200033545494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,255,0.021002667645613354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,255,0.021589333812395733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,511,0.02805333336194356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,1023,0.05861866474151611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,1023,0.042549331982930504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,2047,0.1009333332379659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,2047,0.06684266527493794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,3,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,float16,4095,0.1835626761118571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,64,8,128,1,float16,fp8,4095,0.11141332983970642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,1,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,3,0.011141333729028702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,7,0.011349332829316458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,7,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,15,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,15,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,31,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,31,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,63,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,63,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,127,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,127,0.013477332890033722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,255,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,255,0.013679999858140945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,511,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,511,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,1023,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,1023,0.014426667243242264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,2047,0.013967999567588171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,2047,0.015423999478419622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,4095,0.015530666957298914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,4095,0.016575999557971954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,8191,0.01830400029818217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,8191,0.019823999454577763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,float16,16383,0.022517333428064983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,1,128,1,float16,fp8,16383,0.02465066562096278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,1,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,3,0.01098666712641716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,3,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,31,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,7,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,7,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,15,0.010890666395425797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,15,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,255,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,31,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,63,0.01097600037852923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,63,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,127,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,127,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,255,0.013359999905029932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,511,0.012485332787036896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,511,0.01421333352724711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,4095,0.016997333616018295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,1023,0.013007999708255133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,1023,0.014416000495354334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,2047,0.01451733335852623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,2047,0.015765332927306492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,4095,0.015930666277805965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,8191,0.0184906671444575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,8191,0.019871999820073444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,float16,16383,0.023232000569502514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,2,128,1,float16,fp8,16383,0.024959998826185863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1,0.011221333096424738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,1,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,3,0.011178666104873022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,3,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,7,0.011429333438475927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,7,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,15,0.011381333072980246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,15,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,127,0.013562666873137156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,31,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,31,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,63,0.011343999455372492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,63,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,127,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,1023,0.014837333311637243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,255,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,2047,0.01613333324591319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,255,0.01350933313369751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,511,0.012768000364303589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,511,0.014277332772811254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,1023,0.014074667046467463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,2047,0.015237333873907724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,4095,0.016917333006858826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,4095,0.017344000438849132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,8191,0.018618666877349217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,8191,0.01985599969824155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,float16,16383,0.02385066697994868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,4,128,1,float16,fp8,16383,0.02566933383544286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1,0.011584000041087469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,1,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,3,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,3,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,31,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,7,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,7,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,15,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,15,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,31,0.011642667154471079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,63,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,63,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,127,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,127,0.012879999975363413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,255,0.012821332861979803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,255,0.013989333063364029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,511,0.01351999988158544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,511,0.01469333345691363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,8191,0.01941866676012675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,1023,0.01381333296497663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,1023,0.015392000476519266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,2047,0.015135999768972397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,2047,0.016010666886965435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,4095,0.016895999511082966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,4095,0.017450666675964992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,8191,0.02051199972629547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,float16,16383,0.037647999823093414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,64,8,128,1,float16,fp8,16383,0.026869334280490875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1,0.011312000453472137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,1,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,3,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,3,0.012543999900420507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,31,0.01138666644692421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,7,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,7,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,15,0.011178666104873022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,15,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,31,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,63,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,63,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,127,0.013850666582584381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,127,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,255,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,2047,0.015439999600251516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,255,0.013669333110253016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,511,0.0129120002190272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,511,0.01413333291808764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,1023,0.014106666048367819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,1023,0.014933332800865173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,16383,0.029317334294319153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,2047,0.016069332758585613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,4095,0.017957333475351334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,4095,0.01893866683046023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,3,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,3,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,float16,8191,0.02082666630546252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,8191,0.023386667172114056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,1,128,1,float16,fp8,16383,0.03389333436886469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,1,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,31,0.012885333349307379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,63,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,7,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,7,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,15,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,15,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,255,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,31,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,63,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,1023,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,127,0.01137599969903628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,127,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,255,0.012469333906968435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,4095,0.019695999721686046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,511,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,511,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,1023,0.016650666793187458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,2047,0.01586666703224182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,2047,0.01706133286158244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1,0.011343999455372492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,4095,0.019333332777023315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,8191,0.021930667261282604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,8191,0.024330665667851765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,float16,16383,0.0305226668715477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,2,128,1,float16,fp8,16383,0.03392533212900162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,1,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,3,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,3,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,7,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,7,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,63,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,15,0.01163200040658315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,15,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,31,0.011637333780527115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,31,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,63,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,127,0.011424000064531961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,127,0.013882666826248169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,255,0.012847999731699625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,255,0.014602666099866232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,511,0.01358933374285698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,4095,0.01833600054184596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,511,0.014773332824309668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,1023,0.014869333555301031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,1023,0.015696000307798386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,2047,0.016149333367745083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,2047,0.016938666502634685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,4095,0.019946667055288952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,8191,0.022085333863894146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,8191,0.02378133436044057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,3,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,float16,16383,0.03763733307520548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,4,128,1,float16,fp8,16383,0.035455999275048576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1,0.011733333269755045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,1,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,3,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,7,0.011530666301647821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,7,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,63,0.012527999778588613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,15,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,15,0.012485332787036896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,255,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,31,0.01166933278242747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,31,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,63,0.011637333780527115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,127,0.01163200040658315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,127,0.014282666146755219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,255,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,2047,0.01669866715868314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,511,0.013717333475748697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,511,0.014762666076421738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,1023,0.014885333677132925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,1023,0.016154666741689045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,2047,0.016735999534527462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,4095,0.018325333793958027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,4095,0.01952533299724261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,8191,0.03551466763019562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,8191,0.02458133300145467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,float16,16383,0.05734399954477946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,64,8,128,1,float16,fp8,16383,0.040762667854626976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1,0.012981332838535309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,1,0.014362666755914688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,3,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,3,0.0143306665122509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,7,0.013306666165590286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,31,0.01422400027513504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,7,0.014314666390419006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,15,0.01310933381319046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,15,0.01421333352724711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,31,0.013237333546082178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,63,0.01312000056107839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,63,0.014485333114862442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,127,0.013621332744757334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,511,0.018709332992633183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,127,0.015642666568358738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,255,0.015013333410024643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,255,0.016037333756685257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,511,0.016735999534527462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,1,0.014533333480358124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,1023,0.02390933285156886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,1023,0.023647998770078022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,float16,2047,0.03851199895143509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,1,128,1,float16,fp8,2047,0.03544000039498011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,3,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,3,0.014293332894643148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,7,0.01328533391157786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,7,0.0144213338692983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,15,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,15,0.014671999961137772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,31,0.013343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,31,0.014501333236694336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,63,0.01333333303531011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,63,0.014368000129858652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,127,0.01368533323208491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,127,0.01570133368174235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,255,0.01552533358335495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,255,0.01659199967980385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,511,0.01858666663368543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,511,0.0191040001809597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1,0.017093333105246227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,2047,0.041738669077555336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,1023,0.03655466685692469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,fp8,1023,0.025424001117547352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,2,128,1,float16,float16,2047,0.057061334451039634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,1,0.018592000007629395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,3,0.017050666113694508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,3,0.01841066653529803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,31,0.017082666357358296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,31,0.018485333770513535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,7,0.017082666357358296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,7,0.018405333161354065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,15,0.017344000438849132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,15,0.018357332795858383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,63,0.01720533271630605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,63,0.018426666657129925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,511,0.03882133215665817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,511,0.03072533259789149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,127,0.017386666188637417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,127,0.021231998999913532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,255,0.022090665996074677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,255,0.0225600004196167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,1023,0.06081066528956095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,1023,0.044938668608665466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,float16,2047,0.10179199775060017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,4,128,1,float16,fp8,2047,0.06920533378918965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1,0.02342933416366577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,1,0.02531733363866806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,3,0.024218666056791942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,3,0.02499199906984965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,7,0.023605334262053173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,7,0.024885334074497223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,15,0.023898666103680927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,15,0.02493866781393687
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,31,0.023557332654794056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,31,0.02518400053183238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,63,0.024490666886170704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,63,0.02515200028816859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,127,0.025962665677070618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,127,0.030063999195893604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,255,0.040922666589419045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,255,0.03410666684309641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,511,0.06176533301671346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,511,0.05101866523424784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,1023,0.10335466265678406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,1023,0.0732426643371582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,float16,2047,0.18582934141159058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1,0.010816000401973724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,64,8,128,1,float16,fp8,2047,0.11972266435623169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,1,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,3,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,3,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,7,0.010901333143313726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,7,0.01181866725285848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,15,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,15,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,31,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,31,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,63,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,63,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,511,0.014650666465361914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,127,0.010698666175206503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,127,0.013301332791646322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,255,0.012229333321253458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,255,0.013594667116800943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,511,0.013546666751305262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,1023,0.015125333021084467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,1023,0.015770666301250458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,8191,0.032602667808532715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,2047,0.015989333391189575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,2047,0.018181333939234417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,4095,0.02059200033545494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,4095,0.022634667654832203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,8191,0.02914133419593175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,float16,16383,0.04786666731039683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,1,128,1,float16,fp8,16383,0.05236800014972687
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,1,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,3,0.011061333119869232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,3,0.011855999628702799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,7,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,7,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,15,0.011488000551859537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,15,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,31,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,31,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,63,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,63,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,127,0.011114666859308878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,127,0.013066666821638743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,255,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,255,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,511,0.01381333296497663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,511,0.015418666104475657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,1023,0.015520000209410986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,1023,0.016384000579516094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,2047,0.016447999825080235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,2047,0.019071999937295914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,4095,0.02139200021823247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,4095,0.023797333240509033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,3,0.011685332904259363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,8191,0.031034665803114574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,3,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,8191,0.032858667274316154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,float16,16383,0.05091733237107595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,2,128,1,float16,fp8,16383,0.053871999184290566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1,0.011557333171367645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,1,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,63,0.011760000139474869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,7,0.011695999652147293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,7,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,15,0.011717333147923151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,15,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,31,0.011413333316644033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,31,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,63,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,127,0.011498666057984034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,127,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,255,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,2047,0.017269333203633625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,255,0.014885333677132925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,511,0.014325333138306936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,511,0.016048000504573185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,1023,0.015647999942302704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,8191,0.035391998787721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,16383,0.061621333161989846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,1023,0.016805333395799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,2047,0.018885333091020584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,3,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,4095,0.022645334402720135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,4095,0.023210667073726654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,float16,8191,0.03833599885304769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,4,128,1,float16,fp8,16383,0.05783999959627787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,1,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,3,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,7,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,7,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,15,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,15,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,31,0.01163200040658315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,31,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,63,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,63,0.012469333906968435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,127,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,127,0.01394133393963178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,255,0.013605333864688873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,255,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,511,0.014682666709025701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,511,0.01552533358335495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,1023,0.016554666062196095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,1023,0.016480000068744022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,2047,0.017429333180189133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,2047,0.01883200059334437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,4095,0.03660800059636434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,4095,0.023567999402681988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,8191,0.05780800183614095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,1,0.016688000410795212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,8191,0.04507733384768168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,float16,16383,0.10095999638239543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,64,8,128,1,float16,fp8,16383,0.06459199885527293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1,0.015562667200962702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,3,0.015674666812022526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,3,0.016821333517630894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,7,0.015493333339691162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,7,0.016751999656359356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,15,0.015728000551462173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,15,0.01708799973130226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,31,0.01568000018596649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,31,0.017029333859682083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,63,0.015706667055686314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,63,0.016837333639462788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,127,0.01594666639963786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,127,0.017968000223239262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,255,0.017909333109855652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,255,0.019152000546455383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,511,0.022367998957633972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,511,0.021541332205136616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,float16,1023,0.03955733279387156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,1,128,1,float16,fp8,1023,0.029866665601730347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1,0.018965333700180054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,1,0.020128000527620316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,3,0.019258666783571243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,3,0.020661332954963047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,7,0.0191040001809597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,7,0.020448000480731327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,15,0.018992000569899876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,15,0.020527999848127365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,31,0.01893866683046023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,31,0.02027733375628789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,63,0.019258666783571243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,63,0.020821332931518555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,127,0.01974933346112569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,127,0.023130667706330616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,255,0.025610665480295818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,255,0.024421334266662598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,511,0.04197866717974345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,511,0.03356266766786575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,float16,1023,0.06393066545327504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,2,128,1,float16,fp8,1023,0.04897066454092661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1,0.02605866640806198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,1,0.0276853342851003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,3,0.025834667185942333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,3,0.027776000400384266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,7,0.025685332715511322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,7,0.027445333699385326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,15,0.026234666506449383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,15,0.028069332242012024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,31,0.026154667139053345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,31,0.02779199928045273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,63,0.025978667040665943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,63,0.027690666417280834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,127,0.02977066735426585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,127,0.031888000667095184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,255,0.04513066510359446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,1023,0.10710933804512024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,255,0.03853866706291834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,float16,511,0.06746666630109151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,511,0.05444799860318502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,4,128,1,float16,fp8,1023,0.07705600063006084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1,0.038912000755469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,1,0.03991466760635376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,3,0.03841600070397059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,7,0.03898133337497711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,3,0.04012266546487808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,7,0.04016000032424927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,15,0.03824000060558319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,31,0.03938133269548416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,63,0.040549332896868386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,63,0.04050133377313614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,31,0.04012266546487808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,15,0.04027733455101649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,127,0.04711466530958811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,127,0.05132266879081726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,255,0.06631466746330261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,255,0.06472533444563548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,1023,0.13006933530171713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,511,0.10905067125956218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,float16,1023,0.19029333194096884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,64,8,128,1,float16,fp8,511,0.0858133335908254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,1,0.024405332903067272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,1,0.025674665967623394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,3,0.024170666933059692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,7,0.024165332317352295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,7,0.026149332523345947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,3,0.02589333305756251
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,15,0.024256000916163128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,15,0.025909334421157837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,31,0.024453334510326385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,31,0.025594666600227356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,63,0.02436800052722295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,63,0.02608533451954524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,127,0.025685332715511322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,127,0.0281333327293396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,1,0.03033600002527237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,float16,255,0.033914667864640556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,1,128,1,float16,fp8,255,0.030229332546393078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,1,0.032399999598662056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,3,0.032645332316557564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,15,0.030389333764712017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,3,0.030752000709374745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,7,0.030586667358875275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,7,0.032101333141326904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,15,0.0323786661028862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,31,0.031082667410373688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,63,0.03169066707293192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,31,0.03239466746648153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,63,0.03238933285077413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,127,0.03825066735347112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,127,0.03754666695992152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,float16,255,0.051551997661590576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,1,0.04407466451327006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,2,128,1,float16,fp8,255,0.0469760000705719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,1,0.04685866832733154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,3,0.04318933188915253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,3,0.04647466540336609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,7,0.04389866689840952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,7,0.04678399860858917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,15,0.04369066655635834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,15,0.04673066735267639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,31,0.04401599864164988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,31,0.047007997830708824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,63,0.049029335379600525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,63,0.04677866895993551
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,127,0.055200000603993736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,127,0.05916266640027364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,float16,255,0.0772266685962677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,4,128,1,float16,fp8,255,0.07106133302052815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,1,0.06766400237878163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,1,0.07187200089295705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,3,0.06758399804433186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,3,0.0713973343372345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,7,0.06919466455777486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,7,0.07177066802978516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,15,0.06814399858315785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,31,0.07214933137098949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,15,0.07145066559314728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,31,0.07214933137098949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,63,0.07436266541481018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,63,0.07566933333873749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,127,0.0835093359152476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,127,0.09409067034721375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,float16,255,0.1197760005791982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,64,8,128,1,float16,fp8,255,0.1104746659596761
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,1,0.04783466458320618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,1,0.04938666522502899
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,3,0.047498668233553566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,3,0.04896000027656555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,7,0.047968000173568726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,7,0.04953599969546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,15,0.047600001096725464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,15,0.048885335524876915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,31,0.04814399778842926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,31,0.04951466619968414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,63,0.0497920016447703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,63,0.04941866795221964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,float16,127,0.05751466751098633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,1,128,1,float16,fp8,127,0.05410666763782501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,1,0.059301331639289856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,1,0.06158933540185293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,3,0.05994133154551188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,7,0.0590826670328776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,7,0.061861331264177956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,3,0.06235733131567637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,15,0.06001066664854685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,15,0.062394668658574425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,31,0.06100266675154368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,31,0.061754668752352394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,63,0.063701331615448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,63,0.0680266668399175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,float16,127,0.07196799914042155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,2,128,1,float16,fp8,127,0.07745600243409474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,1,0.08619733651479085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,1,0.09135466814041138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,3,0.08573866883913676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,3,0.08999466896057129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,7,0.08564266562461853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,7,0.09107200304667155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,15,0.08845866719881694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,15,0.09172800183296204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,31,0.09292266766230266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,31,0.09322133660316467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,63,0.0934986670811971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,63,0.09725333253542583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,float16,127,0.10019200046857198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,1,0.1338879962762197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,4,128,1,float16,fp8,127,0.11450666189193726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,1,0.1411893367767334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,3,0.13593066732088724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,3,0.1395039955774943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,7,0.13597333431243896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,15,0.13873599966367087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,7,0.1398293375968933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,15,0.14249066511789957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,31,0.1431946655114492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1,0.011168000598748526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,1,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,31,0.14636266231536865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,3,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,3,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,7,0.011381333072980246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,63,0.1439786652723948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,15,0.011242666592200598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,15,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,31,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,63,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,63,0.1474293371041616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,7,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,63,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,127,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,31,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,127,0.013349333157142004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,255,0.012602667013804117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,255,0.014453332871198654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,1023,0.017952000101407368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,511,0.014495999862750372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,float16,127,0.15628266334533691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,64,8,128,1,float16,fp8,127,0.18154666821161905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,511,0.015605332950750986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,1023,0.016106666376193363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,2047,0.020319999506076176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,2047,0.023141334454218548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,4095,0.028912000358104706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,4095,0.032831999162832894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,16383,0.08369599779446919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,float16,8191,0.04859733581542969
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,16383,0.096778670946757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,1,128,1,float16,fp8,8191,0.053258667389551796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,1,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,3,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,3,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,31,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,7,0.011749333391586939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,7,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,15,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,15,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,31,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,63,0.01157333329319954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,63,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,127,0.011786667009194693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,127,0.01339200014869372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,255,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,2047,0.02086399992307027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,255,0.015861333658297855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,511,0.014858666807413101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,511,0.01605333387851715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,1023,0.01618133361140887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,1023,0.018672000616788864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,2047,0.02319466571013133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,4095,0.03166399896144867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,1,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,4095,0.033376000821590424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,8191,0.05150933563709259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,8191,0.05519466598828634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,float16,16383,0.08454400300979614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,2,128,1,float16,fp8,16383,0.10009066263834636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,15,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,3,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,3,0.0129120002190272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,7,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,7,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,15,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,31,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,31,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,63,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,63,0.012954667210578918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,127,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,127,0.014149333039919535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,255,0.01370666672786077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,255,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,511,0.014757333944241205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,511,0.015967999895413715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,1023,0.017050666113694508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,1023,0.01887999971707662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,2047,0.0225600004196167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,2047,0.02364266663789749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,4095,0.03835733234882355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,4095,0.03572800010442734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,8191,0.07196799914042155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,8191,0.05824000140031179
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,float16,16383,0.10902933279673259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,4,128,1,float16,fp8,16383,0.09976533055305481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,1,0.012879999975363413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,3,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,3,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,7,0.012485332787036896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,7,0.013050666699806849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,15,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,63,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,15,0.012986666212479273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,31,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,255,0.014368000129858652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,31,0.012826666235923767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,63,0.012453333785136541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,127,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,1023,0.018506667266289394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,1023,0.018719999740521114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,127,0.014378666877746582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,255,0.015824000040690105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,4095,0.041034666200478874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,8191,0.09993599851926167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,511,0.015589332828919092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,511,0.017173333714405697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,2047,0.03642133375008901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,16383,0.12619733810424805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,2047,0.025610665480295818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,4095,0.05885333319505056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,fp8,8191,0.06364800035953522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,64,8,128,1,float16,float16,16383,0.18445332845052084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,1,0.041482667128245033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,1,0.03884266565243403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,3,0.04385599990685781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,7,0.04342933495839437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,3,0.0394400010506312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,15,0.04725333551565806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,7,0.03992533435424169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,15,0.04141333450873693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,31,0.047797332207361855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,31,0.042064001162846885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,fp8,63,0.047775998711586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,1,128,1,float16,float16,63,0.04795733094215393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,1,0.07050666709740956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,1,0.06303999821345012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,3,0.07186133166154225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,3,0.06471466521422069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,7,0.0734986662864685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,7,0.06515733400980632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,15,0.07467199862003326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,15,0.06665066878000896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,31,0.07530666887760162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,31,0.07307733098665874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,float16,63,0.07646400233109792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,2,128,1,float16,fp8,63,0.07478933533032735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,1,0.12484266360600789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,1,0.1127306620279948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,3,0.12684800227483115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,3,0.11479999621709187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,7,0.12971199552218118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,7,0.11875200271606445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,15,0.1307413379351298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,15,0.12548800309499106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,31,0.1307146648565928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,31,0.12594133615493774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,float16,63,0.13236799836158752
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,1,0.20548266172409058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,1,0.20367467403411865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,4,128,1,float16,fp8,63,0.12769066294034323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,3,0.20850133895874023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,3,0.2071253259976705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,7,0.20709333817164102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,7,0.21244800090789795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,15,0.21654399236043295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,15,0.21808000405629477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,31,0.2216320037841797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,31,0.21571199099222818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,float16,63,0.22985599438349405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,64,8,128,1,float16,fp8,63,0.22180267175038657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,1,0.0796800007422765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,1,0.08190933366616567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,3,0.0828906645377477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,7,0.08132799963156383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,3,0.08025066554546356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,7,0.08015466729799907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,15,0.0817333310842514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,15,0.08020266890525818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,float16,31,0.08075200021266937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,1,0.12644267082214355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,1,128,1,float16,fp8,31,0.0825493335723877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,3,0.13525866468747458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,1,0.13371733824412027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,7,0.13540800412495932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,15,0.12970667084058127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,3,0.1262079974015554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,15,0.13431466619173685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,float16,31,0.13518399993578592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,31,0.13128532965977988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,2,128,1,float16,fp8,7,0.12794133027394614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,1,0.2421813408533732
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,1,0.23278933763504028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,3,0.2439840038617452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,3,0.23174399137496948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,7,0.2424959937731425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,7,0.23407999674479166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,15,0.24706665674845377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,15,0.2339786688486735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,float16,31,0.2479626735051473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,1,0.4046880006790161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,4,128,1,float16,fp8,31,0.23924267292022705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,1,0.4028960069020589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,3,0.40150400002797443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1,0.011760000139474869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,3,0.4054826498031616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,7,0.4167413314183553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,7,0.4039626518885295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,1,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,3,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,3,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,7,0.011663999408483505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,7,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,15,0.011695999652147293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,15,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,31,0.012522666404644648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,31,0.011744000017642975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,63,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,15,0.42494932810465497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,63,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,127,0.011477333803971609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,127,0.013471999516089758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,255,0.012645332763592402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,15,0.4117973248163859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,511,0.015189333508412043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,255,0.014677333335081736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,511,0.016821333517630894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,1023,0.018981333822011948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,1023,0.021482666333516438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,float16,31,0.4322933355967204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,64,8,128,1,float16,fp8,31,0.41484800974527997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,2047,0.031930667658646904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,2047,0.027930667002995808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,4095,0.046767999728520714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,4095,0.050213331977526345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,float16,8191,0.08137066662311554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,1,128,1,float16,fp8,8191,0.09548266728719075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,1,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,3,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,3,0.012874666601419449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,7,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,7,0.012784000486135483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,15,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,15,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,127,0.013983999689420065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,31,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,31,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,63,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,63,0.01393066719174385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,127,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,255,0.01440000037352244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,255,0.01522133375207583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,511,0.015301333119471868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,511,0.017711999515692394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,1023,0.020138667275508244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,1023,0.02222399910291036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,2047,0.02945599953333537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,2047,0.032431999842325844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,4095,0.0499893327554067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,3,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,4095,0.052426666021347046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,float16,8191,0.08238933483759563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,2,128,1,float16,fp8,8191,0.10005866487820943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,1,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,3,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,7,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,7,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,15,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,15,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,31,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,31,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,63,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,63,0.013354666531085968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,127,0.012794667234023413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,127,0.014848000059525171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,255,0.014015999933083853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,255,0.014975999792416891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,511,0.01562133307258288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,511,0.017562666287024815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,1023,0.022874665757020313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,1023,0.022229333718617756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,2047,0.03697066754102707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,2047,0.03412266572316488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,4095,0.05816533168156942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,4095,0.05719999969005585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,float16,8191,0.10042666395505269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,4,128,1,float16,fp8,8191,0.09884799520174663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1,0.012549333274364471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,1,0.013477332890033722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,3,0.012453333785136541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,3,0.013295999417702356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,7,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,7,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,15,0.012421333541472753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,15,0.013269333789745966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,31,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,31,0.01339200014869372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,63,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,63,0.013482666263977686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,127,0.012954667210578918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,127,0.014864000181357065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,1023,0.0234400009115537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,2047,0.05603733162085215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,255,0.014741333822409311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,255,0.01552533358335495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,511,0.017621333400408428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,511,0.01815466706951459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,1023,0.03549333413441976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,2047,0.040106666584809623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,4095,0.09783466657002766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,4095,0.07088000078996022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,float16,8191,0.18356800079345703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,64,8,128,1,float16,fp8,8191,0.10895466804504395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,1,0.012991999586423239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,3,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,31,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,3,0.012826666235923767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,7,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,7,0.012703999876976013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,15,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,15,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,255,0.013898666948080063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,31,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,63,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,63,0.0129120002190272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,127,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,127,0.014122666170199713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,255,0.01469333345691363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,511,0.015834666788578033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,511,0.017498667041460674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,1023,0.02027733375628789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,1023,0.02222399910291036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,2047,0.029733332494894665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,2047,0.03243733445803324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,float16,4095,0.047797332207361855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,1,128,1,float16,fp8,4095,0.052629331747690834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,1,0.012714666624863943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,3,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,3,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,63,0.012634667257467905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,7,0.012421333541472753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,7,0.013295999417702356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,15,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,127,0.014677333335081736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,15,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,255,0.014954666296641031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,31,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,31,0.012901333471139273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,63,0.013834666460752487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,127,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,255,0.01413333291808764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,511,0.016165333489576977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,511,0.017711999515692394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,1023,0.021269333859284718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,1023,0.02275199939807256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,2047,0.041146665811538696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,2047,0.03310399999221166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,7,0.012517333030700684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,float16,4095,0.0590826670328776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,2,128,1,float16,fp8,4095,0.05762666463851929
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,15,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1,0.012543999900420507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,1,0.013690666606028875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,3,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,3,0.013637332866589228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,7,0.013776000589132309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,127,0.013045333325862885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,15,0.013973332941532135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,31,0.012602667013804117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,31,0.013728000223636627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,63,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,63,0.013786666095256805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,127,0.014917333920796713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,255,0.014842666685581207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,2047,0.05699733396371206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,255,0.015599999576807022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,511,0.01720533271630605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,511,0.018426666657129925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,1023,0.03509333233038584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,1023,0.02430933217207591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,2047,0.041146665811538696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,float16,4095,0.09948266545931499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,4,128,1,float16,fp8,4095,0.0639573335647583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,7,0.01747200017174085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1,0.01647466669480006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,1,0.01785600061217944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,3,0.016597333053747814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,3,0.017690667261679966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,7,0.016864000509182613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,15,0.016714667280515034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,63,0.017642666896184284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,31,0.016575999557971954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,15,0.017504000415404636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,31,0.017701332767804463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,255,0.02181866765022278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,511,0.03810133288304011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,63,0.01703466723362605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,127,0.016885332763195038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,127,0.020703999946514767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,255,0.020026666422684986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,511,0.02921066681543986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,1023,0.05884266893068949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,1023,0.043882668018341064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,2047,0.10065600275993347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,2047,0.0676746666431427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,float16,4095,0.18556799491246542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,48,8,128,1,float16,fp8,4095,0.11286399761835735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,1,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,3,0.011034666250149408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,3,0.012549333274364471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,7,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,7,0.011007999380429586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,15,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,15,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,31,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,31,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,63,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,63,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,127,0.011114666859308878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,127,0.013359999905029932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,255,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,255,0.013546666751305262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,511,0.012703999876976013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,511,0.013712000101804733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,1023,0.012821332861979803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,1023,0.0144213338692983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,2047,0.013898666948080063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,2047,0.015146666516860327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,4095,0.01562133307258288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,4095,0.01639466608564059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,1,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,8191,0.0176959993938605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,8191,0.019285333653291065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,float16,16383,0.02231466770172119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,1,128,1,float16,fp8,16383,0.02483733246723811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,15,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,15,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,3,0.010960000256697336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,3,0.012421333541472753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,7,0.011018666128317514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,127,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,7,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,255,0.012565333396196365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,31,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,31,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,63,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,63,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,127,0.011039999624093374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,255,0.013359999905029932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,511,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,511,0.013653332988421122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,1023,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,1023,0.014538666854302088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,2047,0.014314666390419006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,2047,0.015861333658297855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,4095,0.01602666700879733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,4095,0.01749333366751671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,8191,0.01859733338157336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,8191,0.019776000330845516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,float16,16383,0.023157333334287006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,2,128,1,float16,fp8,16383,0.025546667476495106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,7,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,1,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,3,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,3,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,63,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,7,0.011477333803971609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,15,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,15,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,31,0.01137599969903628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,31,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,511,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,63,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,1023,0.015466666469971338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,127,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,127,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,255,0.01258133351802826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,255,0.013674666484196981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,511,0.014405333747466406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,1023,0.013861333330472311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,2047,0.015322666615247726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,2047,0.016309333344300587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,4095,0.01629866659641266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,4095,0.017488000293572743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,8191,0.01882133384545644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,3,0.011247999966144562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,8191,0.019978666057189304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,float16,16383,0.027877333263556164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,4,128,1,float16,fp8,16383,0.025455998877684276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,15,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1,0.01128000020980835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,1,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,3,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,63,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,7,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,7,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,127,0.011578666667143503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,15,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,31,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,31,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,63,0.01227733368674914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,511,0.014240000396966934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,127,0.013354666531085968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,255,0.012746666868527731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,255,0.014394666999578476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,511,0.01381333296497663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,1023,0.014303999642531076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,1023,0.01524266724785169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,2047,0.015360000232855478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,2047,0.016255999604860943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,4095,0.016629333297411602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,4095,0.01803733284274737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,8191,0.019493332753578823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,8191,0.02011200040578842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,float16,16383,0.036992001036802925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,48,8,128,1,float16,fp8,16383,0.0262773334980011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1,0.011034666250149408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,1,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,3,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,3,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,7,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,7,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,15,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,15,0.012485332787036896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,31,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,31,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,63,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,63,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,127,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,127,0.0145066666106383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,255,0.012469333906968435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,255,0.013541333377361298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,511,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,511,0.014042666802803675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,1023,0.01393066719174385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,1023,0.015034666905800501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,2047,0.01534933348496755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,2047,0.01602666700879733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,4095,0.017498667041460674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,4095,0.018826667219400406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,8191,0.020954666038354237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,8191,0.02332799881696701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,float16,16383,0.029616000751654308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,1,128,1,float16,fp8,16383,0.03364799916744232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,7,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,1,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,3,0.011039999624093374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,3,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,7,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,15,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,15,0.012373333175977072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,255,0.012522666404644648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,31,0.011141333729028702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,31,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,63,0.011349332829316458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,63,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,127,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,127,0.013189333180586496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,255,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,511,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,511,0.014469332993030548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,8191,0.021759999295075733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,1023,0.014117332796255747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,1023,0.015834666788578033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,2047,0.01591466615597407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,2047,0.01670933390657107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,4095,0.018122666825850803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,4095,0.01966933285196622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,3,0.011391999820868174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,8191,0.02367466688156128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,float16,16383,0.030224000414212544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,2,128,1,float16,fp8,16383,0.03440533330043157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,1,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,3,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,7,0.01157333329319954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,7,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,15,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,15,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,127,0.013760000467300415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,31,0.011584000041087469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,31,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,63,0.011173332730929056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,63,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,127,0.011546666423479715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,255,0.012666666259368261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,255,0.014245333770910898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,2047,0.016517333686351776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,511,0.013642666240533194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,4095,0.01995733380317688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,511,0.014463999619086584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,1023,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,1023,0.015728000551462173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,2047,0.016997333616018295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,4095,0.018565333137909572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,8191,0.024906667570273083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,8191,0.02438933402299881
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,float16,16383,0.04288533329963684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,4,128,1,float16,fp8,16383,0.03669333209594091
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1,0.011477333803971609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,1,0.013760000467300415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,3,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,3,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,7,0.011663999408483505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,7,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,15,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,15,0.012602667013804117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,31,0.011616000284751257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,255,0.014271999398867289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,31,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,63,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,63,0.012746666868527731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,127,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,127,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,255,0.013327999661366144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,511,0.014058666924635569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,4095,0.01905599981546402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,511,0.01458666721979777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,1023,0.01462399959564209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,1023,0.016261332978804905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,2047,0.016629333297411602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,16383,0.04659200211366018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,2047,0.01722666621208191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,4095,0.01993600030740102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,8191,0.03737066686153412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,fp8,8191,0.025685332715511322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,48,8,128,1,float16,float16,16383,0.058005332946777344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,1,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,3,0.012885333349307379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,3,0.014218666901191076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,7,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,7,0.013834666460752487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,63,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,15,0.01301866645614306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,127,0.01358933374285698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,15,0.013717333475748697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,31,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,31,0.013983999689420065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,63,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,127,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,255,0.014757333944241205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,255,0.01597333326935768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,511,0.016943999876578648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,511,0.01836799954374631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,1023,0.02342933416366577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,1023,0.02276266614596049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,float16,2047,0.03691199918588003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,1,128,1,float16,fp8,2047,0.03504000107447306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1,0.01312000056107839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,1,0.0143306665122509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,3,0.013573333621025085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,31,0.013386666774749756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,3,0.01422400027513504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,7,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,7,0.01440000037352244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,15,0.01309866706530253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,15,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,31,0.014432000617186228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,63,0.01339200014869372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,255,0.016293333222468693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,511,0.01798933371901512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,511,0.019093333433071773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,63,0.01452800010641416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,127,0.013610667238632837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,127,0.015397333850463232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,255,0.015392000476519266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,1023,0.03641066700220108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,1023,0.025583999852339428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,float16,2047,0.05791999896367391
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,2,128,1,float16,fp8,2047,0.04127999891837438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,15,0.018357332795858383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,1,0.018122666825850803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,3,0.017045332739750545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,3,0.018357332795858383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,7,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,7,0.01833600054184596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,15,0.017162666966517765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,127,0.017488000293572743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,31,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,255,0.02128533273935318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,31,0.018250666558742523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,63,0.017093333105246227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,63,0.018346666047970455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,127,0.021066665649414062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,255,0.02216000109910965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,511,0.038293334345022835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,511,0.028736000259717304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,1023,0.06035199761390686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,1023,0.045082668463389076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,float16,2047,0.10122133294741313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,4,128,1,float16,fp8,2047,0.06964266796906789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1,0.0239680012067159
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,1,0.02553066611289978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,7,0.023951999843120575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,3,0.024112001061439514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,3,0.025663999219735462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,7,0.025498665869235992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,15,0.02437866727511088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,15,0.025258667767047882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,31,0.02382933348417282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,127,0.030181333422660828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,31,0.025381334125995636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,63,0.024351999163627625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,63,0.025797332326571148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,127,0.02566933383544286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,255,0.04285866518815359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,255,0.035530666510264076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,1023,0.10373866558074951
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,511,0.05117866893609365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,511,0.062021334966023765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,1023,0.07425066828727722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,float16,2047,0.1848906675974528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,3,0.011674666156371435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,48,8,128,1,float16,fp8,2047,0.12131200234095256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,1,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,3,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,7,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,63,0.010885333021481832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,7,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,15,0.01090666651725769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,15,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,31,0.010901333143313726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,255,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,31,0.011557333171367645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,63,0.011706666400035223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,127,0.011039999624093374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,127,0.012944000462690989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,2047,0.01586666703224182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,255,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,511,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,4095,0.020058666666348774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,511,0.01452800010641416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,1023,0.014757333944241205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,1023,0.015466666469971338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,2047,0.018496000518401463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,4095,0.022661333282788593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,8191,0.029002666473388672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,8191,0.0328053335348765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,float16,16383,0.04667200148105621
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,7,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,1,128,1,float16,fp8,16383,0.05197333296140035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1,0.011141333729028702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,1,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,31,0.011002667248249054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,3,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,63,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,3,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,7,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,15,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,255,0.012543999900420507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,15,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,31,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,63,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,127,0.01156266654531161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,127,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,255,0.014175999909639359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,511,0.013738666971524557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,511,0.015392000476519266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,1023,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,8191,0.030234667162100475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,1023,0.01599466676513354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,2047,0.016261332978804905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,2047,0.01921066641807556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1,0.011434666812419891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,4095,0.020869334538777668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,4095,0.023520000278949738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,8191,0.03299200038115183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,float16,16383,0.05050133168697357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,2,128,1,float16,fp8,16383,0.05313600103060404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,1,0.011936000237862269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,3,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,3,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,7,0.011578666667143503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,7,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,15,0.011626667032639185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,15,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,31,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,31,0.011978667229413986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,63,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,63,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,127,0.011754666765530905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,127,0.013669333110253016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,255,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,255,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,511,0.014368000129858652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,511,0.015930666277805965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,1023,0.01603200038274129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,1023,0.016250666230916977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,2047,0.017173333714405697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,2047,0.018661333868900936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,4095,0.02147199958562851
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,4095,0.023381332556406658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,8191,0.03763733307520548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,8191,0.03495999922355016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,3,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,float16,16383,0.06150933106740316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,4,128,1,float16,fp8,16383,0.0582826683918635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1,0.011642667154471079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,1,0.012752000242471695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,3,0.01249066616098086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,7,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,7,0.012725333372751871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,15,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,15,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,31,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,31,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,63,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,63,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,127,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,127,0.014303999642531076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,1023,0.01655999943614006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,255,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,255,0.014426667243242264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,511,0.014890667051076889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,511,0.015861333658297855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,1023,0.016165333489576977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,2047,0.018288000176350277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,16383,0.10019200046857198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,2047,0.01903466631968816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,16383,0.0734506646792094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,4095,0.0364533339937528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,4095,0.02462399999300639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,float16,8191,0.058880001306533813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,48,8,128,1,float16,fp8,8191,0.041589332123597465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1,0.015402667224407196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,7,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,1,0.016117333124081295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,3,0.015253332753976187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,3,0.016421332955360413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,7,0.016447999825080235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,15,0.01658133293191592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,15,0.016117333124081295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,31,0.015370666980743408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,127,0.01552533358335495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,31,0.01623999948302905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,63,0.015253332753976187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,63,0.016250666230916977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,127,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,255,0.018112000077962875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,255,0.01823466643691063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,511,0.021002667645613354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,511,0.021189334491888683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,float16,1023,0.037733333806196846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,1,128,1,float16,fp8,1023,0.02824000020821889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,3,0.019760000209013622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1,0.018719999740521114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,1,0.020138667275508244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,3,0.018746666610240936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,7,0.01894933357834816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,31,0.018911999960740406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,7,0.019999999552965164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,15,0.018661333868900936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,15,0.019695999721686046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,31,0.019909333437681198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,63,0.018853332847356796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,63,0.020981334149837494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,127,0.019578666736682255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,127,0.022448000808556873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,255,0.02383466561635335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,255,0.024330665667851765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,511,0.04141866664091746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,511,0.03070933371782303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,float16,1023,0.06257600088914235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1,0.02555199960867564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,2,128,1,float16,fp8,1023,0.04830400149027506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,1,0.027024000883102417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,3,0.025472000241279602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,3,0.02722666660944621
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,7,0.025536000728607178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,7,0.026885333160559338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,15,0.02719466636578242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,63,0.02569599946339925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,15,0.02588266630967458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,31,0.027162666122118633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,31,0.02535466601451238
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,63,0.027237333357334137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,127,0.02829866607983907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,511,0.06643199920654297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,127,0.03173333406448364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,255,0.04565866788228353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,255,0.03759466608365377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,float16,1023,0.10552533467610677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,511,0.05429866909980774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1,0.03828266759713491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,4,128,1,float16,fp8,1023,0.07746133208274841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,1,0.041402667760849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,3,0.03835200021664301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,3,0.04108799993991852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,15,0.0383093332250913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,7,0.03809600075085958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,7,0.04098133246103922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,31,0.03909866760174433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,31,0.040752001106739044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,63,0.040991999208927155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,63,0.04155733436346054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,127,0.047498668233553566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,15,0.041519999504089355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,127,0.0528106689453125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,255,0.06981333096822102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,255,0.06358933448791504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,1023,0.1918506622314453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,1023,0.13295466701189676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,float16,511,0.11037333806355794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,48,8,128,1,float16,fp8,511,0.09287466605504353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,1,0.022240000466505688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,1,0.024527999262015026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,3,0.022229333718617756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,7,0.022437334060668945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,3,0.024405332903067272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,15,0.022757334013779957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,31,0.02253866692384084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,31,0.02462933212518692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,63,0.02253866692384084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,7,0.024288001159826916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,63,0.024293333292007446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,15,0.023936000963052113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,127,0.023530667026837666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,127,0.027056001126766205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,float16,255,0.02951466788848241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,1,0.028912000358104706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,1,128,1,float16,fp8,255,0.028570666909217834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,3,0.030896000564098358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,1,0.030917334059874218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,3,0.028789333999156952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,7,0.031471999982992806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,7,0.02897600084543228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,15,0.02938666691382726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,15,0.030847998956839245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,31,0.029306667546431225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,31,0.03120533376932144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,63,0.03126933425664902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,63,0.02940800040960312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,127,0.03346133232116699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,127,0.03545066714286804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,float16,255,0.04975999891757965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,2,128,1,float16,fp8,255,0.043237333496411644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,1,0.04231466849644979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,1,0.044821331898371376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,3,0.041749333341916404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,3,0.044213334719340004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,7,0.04199466605981191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,7,0.04452799757321676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,15,0.04214400053024292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,15,0.04450666904449463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,31,0.04257600009441376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,31,0.04505600035190582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,63,0.04619733492533366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,63,0.04447466631730398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,127,0.052928000688552856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,127,0.05713599920272827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,float16,255,0.07364266614119212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,4,128,1,float16,fp8,255,0.0684746652841568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,1,0.06808533271153767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,3,0.06760000189145406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,1,0.07153066496054332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,3,0.07111999889214833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,7,0.06793599824110667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,15,0.0681279997030894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,7,0.07100800176461537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,15,0.0709440012772878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,31,0.0726506660381953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,31,0.07223999996980031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,63,0.07470400134722392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,63,0.0745119998852412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,127,0.0817493349313736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,127,0.09576533238093059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,fp8,255,0.10938133796056111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,48,8,128,1,float16,float16,255,0.1206719974676768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,1,0.04351999859015147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,1,0.04468800127506256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,3,0.04308799902598063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,3,0.04435733457406362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,7,0.043552001317342125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,7,0.04448533554871877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,15,0.04288533329963684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,31,0.04377066592375437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,31,0.04487466812133789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,15,0.04474666714668274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,63,0.04370133578777313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,63,0.044533332188924156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,fp8,127,0.04890666902065277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,1,128,1,float16,float16,127,0.053269331653912864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,1,0.05509866774082184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,1,0.05765866736570994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,3,0.054885332783063255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,3,0.057087997595469155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,7,0.05490666627883911
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,7,0.05716800192991892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,15,0.056847999493281044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,15,0.055626665552457176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,31,0.05765866736570994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,31,0.05762133498986562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,63,0.06213866670926412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,63,0.05774933099746704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,float16,127,0.06839466591676076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,2,128,1,float16,fp8,127,0.07172800103823344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,1,0.08188800017038982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,1,0.08661333719889323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,3,0.08568533261617024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,3,0.08210666477680206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,7,0.08113066852092743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,7,0.08548266688982646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,15,0.08221333225568135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,15,0.08591467142105103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,31,0.09020266930262248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,31,0.08846933643023173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,63,0.08994133273760478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,63,0.09324799974759419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,float16,127,0.09648000200589497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,1,0.1327893336613973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,4,128,1,float16,fp8,127,0.10878933469454448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,1,0.1384106675783793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,3,0.14036267002423605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,3,0.13408533732096353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,7,0.13190933068593344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,7,0.141077329715093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,15,0.14017066359519958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,15,0.14111999670664468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,31,0.14568000038464865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,31,0.14130666851997375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,1,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,7,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,3,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,7,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,63,0.14436266819636026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,3,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,15,0.01098666712641716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,63,0.1487626632054647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,15,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,31,0.011312000453472137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,63,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,fp8,127,0.1801919937133789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,31,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,48,8,128,1,float16,float16,127,0.15312000115712485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,63,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,127,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,127,0.011424000064531961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,255,0.012549333274364471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,255,0.014592000593741735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,511,0.015541333705186844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,511,0.014074667046467463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,1023,0.017978666971127193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,2047,0.02015999952952067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,1023,0.015850666910409927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,2047,0.023013333479563396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,4095,0.02884799987077713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,4095,0.03214933226505915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,8191,0.04757333298524221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,8191,0.052442664901415505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,float16,16383,0.08341866731643677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,1,128,1,float16,fp8,16383,0.09197866916656494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1,0.011472000430027643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,1,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,3,0.011695999652147293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,3,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,7,0.011381333072980246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,63,0.011514666179815928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,63,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,7,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,15,0.011594666788975397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,255,0.012752000242471695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,255,0.015295999745527903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,15,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,511,0.01628799984852473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,31,0.011551999797423681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,31,0.012549333274364471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,127,0.011594666788975397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,127,0.0136266661187013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,511,0.01452800010641416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,4095,0.03299733251333237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,1023,0.016885332763195038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,1023,0.01820266619324684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,2047,0.021136000752449036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,2047,0.023434666295846302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,4095,0.030832000076770782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,8191,0.04880533119042715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,8191,0.05518933137257894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,float16,16383,0.08471999565760295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,2,128,1,float16,fp8,16383,0.10145066181818645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,1,0.012746666868527731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,3,0.01181866725285848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,3,0.012944000462690989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,7,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,7,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,63,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,15,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,15,0.012719999998807907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,31,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,31,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,63,0.012768000364303589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,127,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,127,0.014250667144854864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,1023,0.01685333376129468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,255,0.013471999516089758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,255,0.015178666760524115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,511,0.014826666563749313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,511,0.01611199975013733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,1023,0.018464000274737675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,2047,0.02186133215824763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,2047,0.0230880007147789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,4095,0.03812800099452337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,4095,0.03469866762558619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,8191,0.059978668888409935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,8191,0.057258665561676025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,float16,16383,0.10685867071151733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,4,128,1,float16,fp8,16383,0.10116799672444661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,1,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,3,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,3,0.013013333082199097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,7,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,7,0.012944000462690989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,15,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,15,0.013189333180586496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,31,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,31,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,255,0.014293332894643148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,63,0.011994666109482447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,63,0.012837332983811697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,511,0.01691199963291486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,127,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,127,0.014159999787807465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,255,0.016048000504573185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,511,0.015392000476519266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,1023,0.018191999445358913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,1023,0.018853332847356796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,2047,0.03702933341264725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,2047,0.02739733209212621
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,4095,0.05817066629727682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,4095,0.04159999887148539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,8191,0.1006773312886556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,8191,0.06482666730880737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,float16,16383,0.18462934096654257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,48,8,128,1,float16,fp8,16383,0.1106666624546051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,1,0.03813866774241129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,1,0.038890667259693146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,3,0.03860799968242645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,7,0.03855466594298681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,3,0.03923733284076055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,7,0.03977599988381068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,15,0.04147200038035711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,15,0.03853866706291834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,31,0.04610133171081543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,31,0.03937600056330363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,float16,63,0.04725333551565806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,1,128,1,float16,fp8,63,0.04146666576464971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,3,0.06549866497516632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,1,0.06320533156394958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,1,0.06507733464241028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,3,0.06243733565012614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,7,0.067221333583196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,7,0.06298133234182994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,15,0.07169066866238911
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,15,0.06379733482996623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,31,0.073594664533933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,31,0.06896533568700154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,fp8,63,0.07299200197060902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,2,128,1,float16,float16,63,0.07499200105667114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,1,0.11521066228548686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,1,0.11338133613268535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,3,0.11949867010116577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,3,0.1129919985930125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,7,0.1267573336760203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,7,0.11593600114186604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,15,0.12086932857831319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,31,0.12850133577982584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,31,0.12657066186269125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,63,0.13054399689038595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,fp8,63,0.12761599818865457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,1,0.21850667397181192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,1,0.20670932531356812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,4,128,1,float16,float16,15,0.1276746690273285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,3,0.20753065745035806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,7,0.23162132501602173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,3,0.22977066040039062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,7,0.21471999088923135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,15,0.22658665974934897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,15,0.23171732823053995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,31,0.2336746652921041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,31,0.2225546638170878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,float16,63,0.2339093287785848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,48,8,128,1,float16,fp8,63,0.23300800720850626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,1,0.07282666862010956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,1,0.0791786660750707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,3,0.0774186650911967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,3,0.07634133100509644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,7,0.07728533446788788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,7,0.0767626663049062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,15,0.07770133515199025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,15,0.07797866563002269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,float16,31,0.07899733384450276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,1,0.13064533472061157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,1,128,1,float16,fp8,31,0.07977066437403361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,1,0.12475732962290446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,3,0.13005866607030234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,3,0.12390399972597758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,7,0.132341335217158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,7,0.12587199608484903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,15,0.131632000207901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,15,0.12819733222325644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,float16,31,0.13421866297721863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,1,0.24021865924199423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,1,0.22990934054056802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,2,128,1,float16,fp8,31,0.1293280025323232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,3,0.24371200799942017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,3,0.23165865739186606
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,7,0.24152533213297525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,7,0.23274133602778116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,15,0.24538666009902954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,15,0.23248533407847086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,float16,31,0.24317866563796997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,1,0.4493600130081177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,1,0.42557334899902344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,4,128,1,float16,fp8,31,0.2374346653620402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,3,0.4440693457921346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,3,0.4262186686197917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,1,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,3,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,3,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,7,0.011749333391586939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,7,0.45083733399709064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,7,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,15,0.011829332758982977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,31,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,15,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,31,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,63,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,63,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,127,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,7,0.42404266198476154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,127,0.01350933313369751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,255,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,255,0.014186666657527288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,15,0.44656534989674884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,511,0.015066667149464289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,511,0.016800000021855038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,15,0.4342079957326253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,1023,0.019637333850065868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,1023,0.02145066608985265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,float16,31,0.45073068141937256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,2047,0.03136000037193298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,8191,0.09468799829483032
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,2047,0.02804800122976303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,4095,0.04632000128428141
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,float16,8191,0.08302933474381764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,48,8,128,1,float16,fp8,31,0.4328800042470296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,1,128,1,float16,fp8,4095,0.05101333558559418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,3,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,3,0.012842666357755661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,31,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,1,0.01292266696691513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,7,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,7,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,15,0.012842666357755661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,15,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,31,0.013125333935022354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,63,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,511,0.015754666179418564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,63,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,127,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,1023,0.022416000564893086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,127,0.013983999689420065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,255,0.013909333695967993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,255,0.014671999961137772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,511,0.017429333180189133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,1023,0.020015999674797058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,2047,0.029520000020662945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,2047,0.03236799935499827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,4095,0.04974933465321859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,4095,0.051327998439470925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,float16,8191,0.08402132987976074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,2,128,1,float16,fp8,8191,0.10035199920336406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1,0.012432000289360682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,1,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,3,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,3,0.01313599944114685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,31,0.012853333105643591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,7,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,7,0.01292266696691513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,15,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,15,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,31,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,255,0.01540800059835116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,63,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,63,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,127,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,127,0.014096000542243322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,255,0.014127999544143677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,511,0.01616000011563301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,511,0.018122666825850803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,1023,0.020261333634455998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,1023,0.022218666970729828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,2047,0.036559998989105225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,2047,0.033626665671666466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,8191,0.09819733103116353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,4095,0.06000000238418579
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,3,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,fp8,4095,0.057071998715400696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,4,128,1,float16,float16,8191,0.10640533765157063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1,0.012495999534924826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,3,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,1,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,7,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,7,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,15,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,15,0.013280000537633896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,31,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,31,0.013306666165590286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,63,0.012730666746695837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,63,0.013317332913478216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,127,0.012906666845083237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,127,0.014794666320085526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,255,0.014463999619086584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,255,0.015562667200962702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,511,0.016906666258970898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,511,0.01800000046690305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,1023,0.034917332231998444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,1023,0.024661332368850708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,2047,0.055760001142819725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,2047,0.04035199930270513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,4095,0.09905067086219788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,4095,0.06232533355553945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,float16,8191,0.18307199080785116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,48,8,128,1,float16,fp8,8191,0.10867200295130412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,7,0.012517333030700684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,1,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,31,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,3,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,3,0.012752000242471695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,7,0.012304000556468964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,15,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,15,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,31,0.012949333836634954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,63,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,63,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,127,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,127,0.014240000396966934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,255,0.013744000345468521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,255,0.014645333091417948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,511,0.015498666713635126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,511,0.017525333911180496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,1023,0.020090666910012562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,1023,0.02236266682545344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,2047,0.029904000461101532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,2047,0.032586666444937386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,float16,4095,0.04956800242265066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,3,0.012784000486135483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,1,128,1,float16,fp8,4095,0.05369600156943003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,1,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,3,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,7,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,7,0.012885333349307379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,15,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,15,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,31,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,31,0.012608000387748083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,63,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,63,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,127,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,127,0.014277332772811254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,255,0.014432000617186228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,255,0.01471466695268949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,511,0.015568000574906668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,511,0.01782400036851565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,1023,0.02111999938885371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,1023,0.022554665803909302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,2047,0.035786665976047516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,2047,0.03340800106525421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,float16,4095,0.058970664938290916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,2,128,1,float16,fp8,4095,0.05663999915122986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,1,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,3,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,3,0.013610667238632837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,7,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,7,0.013445333888133367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,15,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,15,0.013450667262077332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,31,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,31,0.013429333766301474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,63,0.012879999975363413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,63,0.013471999516089758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,127,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,127,0.014650666465361914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,511,0.01844266677896182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,255,0.014842666685581207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,255,0.01553600033124288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,511,0.017418666432301205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,1023,0.035045333206653595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,1023,0.024432001014550526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,2047,0.056890666484832764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,2047,0.04155199974775314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,3,0.016384000579516094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,float16,4095,0.09896533687909444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1,0.016437333077192307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,4,128,1,float16,fp8,4095,0.0645546664794286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,1,0.01730666682124138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,3,0.017653333644072216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,7,0.016613333175579708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,31,0.017690667261679966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,7,0.01801066721479098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,15,0.01642666632930438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,15,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,31,0.016528000434239704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,63,0.01658133293191592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,63,0.01752000053723653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,511,0.03760000069936117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,127,0.01701333373785019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,127,0.020703999946514767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,255,0.020794666061798733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,255,0.021674667795499165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,511,0.03010133405526479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,1023,0.05893866717815399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,1023,0.04400533437728882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,2047,0.10043199857076009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,2047,0.06779199838638306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,float16,4095,0.1840320030848185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,7,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,40,8,128,1,float16,fp8,4095,0.11315199732780457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1,0.010992000500361124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,15,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,1,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,3,0.010816000401973724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,3,0.011978667229413986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,7,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,15,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,31,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,255,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,31,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,63,0.01128000020980835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,63,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,127,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,127,0.013397333522637686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,255,0.013552000125249227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,511,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,511,0.013669333110253016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,4095,0.01626666635274887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,1023,0.012741333494583765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,1023,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,2047,0.013807999591032663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,2047,0.015013333410024643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,4095,0.015477333217859268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,8191,0.017808000246683758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,3,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,8191,0.01899733394384384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,float16,16383,0.022069332500298817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,1,128,1,float16,fp8,16383,0.024432001014550526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,1,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,3,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,31,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,7,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,7,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,15,0.011039999624093374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,15,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,31,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,63,0.011018666128317514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,63,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,511,0.013616000612576803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,127,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,1023,0.014522666732470194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,127,0.013653332988421122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,255,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,255,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,511,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,8191,0.018629333625237148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,1023,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,2047,0.014101333916187286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,2047,0.015978666643301647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,4095,0.016634666671355564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,4095,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,8191,0.019909333437681198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,3,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,float16,16383,0.023103999594847362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,2,128,1,float16,fp8,16383,0.02499733368555705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,15,0.011424000064531961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1,0.011194666226704916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,1,0.012373333175977072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,3,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,7,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,63,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,7,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,15,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,31,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,31,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,255,0.013386666774749756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,63,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,127,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,127,0.013167999684810638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,255,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,511,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,511,0.01434133326013883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,1023,0.013807999591032663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,1023,0.015210667004187902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,2047,0.014901333798964819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,2047,0.016458666572968166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,4095,0.016149333367745083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,4095,0.017024000485738117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,8191,0.019130667050679524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,8191,0.019839999576409657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,float16,16383,0.024106666445732117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,4,128,1,float16,fp8,16383,0.02565866708755493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,1,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,3,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,3,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,7,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,7,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,15,0.011381333072980246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,15,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,31,0.011498666057984034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,31,0.012517333030700684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,63,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,63,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,511,0.014602666099866232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,1023,0.014117332796255747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,127,0.01163200040658315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,127,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,255,0.01258133351802826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,255,0.014277332772811254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,511,0.01350933313369751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,1023,0.015135999768972397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,2047,0.01540800059835116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,2047,0.016858667135238647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,4095,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,4095,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,8191,0.02088533341884613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,8191,0.020261333634455998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,float16,16383,0.03654933224121729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,40,8,128,1,float16,fp8,16383,0.026186667382717133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,1,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,3,0.011221333096424738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,31,0.011343999455372492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,31,0.012543999900420507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,3,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,7,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,127,0.010954666882753372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,7,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,15,0.010970667004585266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,15,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,63,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,63,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,127,0.013354666531085968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,255,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,255,0.01332266628742218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,511,0.012634667257467905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,4095,0.01740266631046931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,511,0.013872000078360239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,1023,0.013552000125249227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,1023,0.014997333288192749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,2047,0.015263999501864115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,2047,0.01603200038274129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1,0.011482667177915573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,4095,0.018458666900793713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,8191,0.020448000480731327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,8191,0.023226665953795116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,float16,16383,0.029205332199732464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,1,128,1,float16,fp8,16383,0.0330079992612203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,1,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,3,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,3,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,7,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,7,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,15,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,127,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,127,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,15,0.01227733368674914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,31,0.011215999722480774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,511,0.012842666357755661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,31,0.012794667234023413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,63,0.011050666371981302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,63,0.012421333541472753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,255,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,2047,0.016458666572968166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,255,0.013690666606028875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,511,0.014357333381970724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,1023,0.013781332721312841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,1023,0.01573333392540614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,2047,0.016229332735141117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,4095,0.01817600056529045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,16383,0.03379733363787333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,4095,0.01913600042462349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,8191,0.021125334004561108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,fp8,8191,0.02386133372783661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,2,128,1,float16,float16,16383,0.030053332448005676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,7,0.011242666592200598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1,0.011551999797423681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,1,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,3,0.011253333340088526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,3,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,7,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,63,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,15,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,15,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,31,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,31,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,63,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,127,0.011584000041087469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,127,0.013429333766301474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,255,0.014165333161751429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,255,0.012624000509579977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,511,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,511,0.014783999572197596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,1023,0.015125333021084467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,1023,0.015834666788578033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,2047,0.016000000139077503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,2047,0.016789333273967106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,4095,0.018298666924238205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,4095,0.019365333020687103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,8191,0.022330666581789654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,8191,0.02422400067249934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,float16,16383,0.03796799977620443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,4,128,1,float16,fp8,16383,0.036090667049090065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1,0.01166933278242747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,15,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,1,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,3,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,3,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,7,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,7,0.0129120002190272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,15,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,31,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,31,0.012709333250919977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,63,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,63,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,127,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,127,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,255,0.013157332936922709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,2047,0.01681600014368693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,255,0.014405333747466406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,511,0.013562666873137156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,511,0.015290666371583939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,1023,0.015168000012636185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,1023,0.015706667055686314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,2047,0.016997333616018295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,4095,0.01972266659140587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,4095,0.019567999988794327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,8191,0.036901332437992096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,8191,0.026693334182103474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,float16,16383,0.058277333776156105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,40,8,128,1,float16,fp8,16383,0.046298667788505554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1,0.012885333349307379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,1,0.013679999858140945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,3,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,3,0.013359999905029932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,31,0.013082666943470636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,7,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,7,0.013647999614477158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,15,0.013077333569526672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,15,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,127,0.015061333775520325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,31,0.01360000049074491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,63,0.013157332936922709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,63,0.013514666507641474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,127,0.012986666212479273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,255,0.014741333822409311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,255,0.01578666642308235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,511,0.016714667280515034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,511,0.018309333672126133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,2047,0.034314667185147606
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,1023,0.020970667401949566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,fp8,1023,0.023386667172114056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,1,128,1,float16,float16,2047,0.036848001182079315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1,0.013237333546082178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,7,0.01443733274936676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,1,0.01443733274936676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,3,0.013194666554530462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,3,0.014416000495354334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,7,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,15,0.013258667041858038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,15,0.01422400027513504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,31,0.013141332815090815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,31,0.014394666999578476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,63,0.013386666774749756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,63,0.014325333138306936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,511,0.018191999445358913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,127,0.013359999905029932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,127,0.015471999843915304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,255,0.015546667079130808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,255,0.01632000009218852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,1023,0.025727999707063038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,511,0.01907733331123988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,1023,0.03623999903599421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,float16,2047,0.05669866502285004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,3,0.018405333161354065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,2,128,1,float16,fp8,2047,0.04168533285458883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,1,0.017957333475351334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,3,0.016842667013406754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,7,0.017093333105246227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,7,0.01820266619324684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,15,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,15,0.01823466643691063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,31,0.016949333250522614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,127,0.017504000415404636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,127,0.02096533278624217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,31,0.018031999468803406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,63,0.017024000485738117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,511,0.038917332887649536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,63,0.018330667167901993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,255,0.022111999491850536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,255,0.022458667556444805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,511,0.03156266609827677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,1023,0.059418668349583946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,1023,0.044922664761543274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,3,0.023904000719388325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,float16,2047,0.10109866658846538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,4,128,1,float16,fp8,2047,0.06885866820812225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1,0.02403733382622401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,15,0.024432001014550526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,1,0.025621332228183746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,3,0.025424001117547352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,7,0.023786666492621105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,7,0.025216000775496166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,15,0.02550933261712392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,31,0.02390933285156886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,31,0.02534399926662445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,63,0.024341332415739696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,63,0.025701334079106648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,127,0.025589334468046825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,127,0.03025600065787633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,255,0.042021334171295166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,255,0.03665599972009659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,511,0.06283733248710632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,511,0.05056533217430115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,1023,0.10270933310190837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,1023,0.07341866691907246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,float16,2047,0.18554667631785074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1,0.010874666273593903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,40,8,128,1,float16,fp8,2047,0.12123200297355652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,1,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,3,0.010970667004585266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,3,0.011530666301647821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,7,0.010922666639089584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,7,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,15,0.010746666540702185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,127,0.011018666128317514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,127,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,255,0.011850666254758835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,15,0.011472000430027643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,31,0.01062400018175443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,31,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,63,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,63,0.011557333171367645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,2047,0.01573866605758667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,255,0.01341333364446958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,511,0.013343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,511,0.014373333503802618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,1023,0.014597332725922266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,8191,0.032885332902272545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,1023,0.015573333948850632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,2047,0.018031999468803406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,4095,0.019989332805077236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,3,0.011077333241701126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,3,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,4095,0.022645334402720135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,8191,0.02937600016593933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,float16,16383,0.04625066618124644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,1,128,1,float16,fp8,16383,0.052426666021347046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,1,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,31,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,7,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,7,0.011701333026091257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,15,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,15,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,31,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,255,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,511,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,63,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,1023,0.015765332927306492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,63,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,127,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,127,0.013023999830087027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,255,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,511,0.015605332950750986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,1023,0.016127999871969223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,2047,0.016415999581416447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,2047,0.01882133384545644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,4095,0.020746666938066483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,4095,0.023541333774725597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,8191,0.029733332494894665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,8191,0.033471999069054924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,3,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,float16,16383,0.05063466727733612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,2,128,1,float16,fp8,16383,0.053898667295773826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,7,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,15,0.011589333415031433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1,0.011594666788975397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,1,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,3,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,63,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,7,0.011600000162919363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,15,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,31,0.011391999820868174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,31,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,63,0.011663999408483505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,127,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,1023,0.01624533285697301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,127,0.013701333353916803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,255,0.013552000125249227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,255,0.014490666488806406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,511,0.01452800010641416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,4095,0.02399466683467229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,511,0.01534933348496755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,8191,0.034917332231998444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,1023,0.016106666376193363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,16383,0.05996799965699514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,2047,0.01655999943614006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,2047,0.018661333868900936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,3,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,4095,0.022410665949185688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,float16,8191,0.03822399924198786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,7,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,4,128,1,float16,fp8,16383,0.05750933289527893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1,0.011551999797423681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,1,0.013066666821638743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,31,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,3,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,7,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,15,0.01181866725285848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,15,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,31,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,63,0.011594666788975397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,63,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,127,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,127,0.014362666755914688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,255,0.013418667018413544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,255,0.014159999787807465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,511,0.014890667051076889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,511,0.017317333569129307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,1023,0.015754666179418564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,1023,0.01653333380818367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,2047,0.017952000101407368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,2047,0.01889066646496455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,16383,0.06524799764156342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,4095,0.03702399879693985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,4095,0.024256000916163128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,8191,0.058229332168896995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,fp8,8191,0.04146666576464971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,3,0.014938666174809137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,40,8,128,1,float16,float16,16383,0.0990613301595052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1,0.014837333311637243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,1,0.01603200038274129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,3,0.01584533353646596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,7,0.014922666052977243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,7,0.016037333756685257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,15,0.015205333630243937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,15,0.01599466676513354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,31,0.014858666807413101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,31,0.01568000018596649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,63,0.014826666563749313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,63,0.015872000406185787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,127,0.015471999843915304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,511,0.02056533346573512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,127,0.017322666943073273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,255,0.017525333911180496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,255,0.018122666825850803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,511,0.020448000480731327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,float16,1023,0.03718400001525879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,1,128,1,float16,fp8,1023,0.027029333015282948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1,0.018383999665578205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,7,0.018330667167901993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,1,0.019648000597953796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,3,0.01858666663368543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,15,0.018415999909241993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,3,0.019797333826621372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,7,0.019461333751678467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,15,0.019440000255902607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,31,0.01841066653529803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,63,0.019706666469573975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,127,0.018719999740521114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,31,0.019621333728233974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,63,0.01855466639002164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,127,0.02248000105222066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,255,0.022458667556444805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,255,0.02370133250951767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,511,0.0414986660083135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,511,0.030095999439557392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,float16,1023,0.06201066573460897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,2,128,1,float16,fp8,1023,0.0473280002673467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1,0.025301332275072735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,1,0.02678400029738744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,3,0.025418666501839954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,3,0.02664000044266383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,7,0.026911998788515728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,7,0.025439999997615814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,15,0.02514133354028066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,15,0.02719466636578242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,31,0.025445332129796345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,31,0.02677333354949951
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,63,0.025237334271272022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,63,0.027130665878454845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,127,0.028336000939210255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,127,0.03124266614516576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,255,0.04451733330885569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,511,0.06463466584682465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,255,0.03762666632731756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,1,0.04073066761096319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1,0.03839466720819473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,511,0.053269331653912864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,3,0.03830400109291077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,float16,1023,0.10562666257222493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,3,0.04049066702524821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,7,0.04068266600370407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,15,0.038586666186650596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,7,0.038005332152048744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,4,128,1,float16,fp8,1023,0.07567999760309856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,31,0.03858133405447006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,63,0.040976000328858696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,15,0.040720000863075256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,63,0.04079466561476389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,31,0.04061333338419596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,127,0.05299200117588043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,127,0.047584002216657005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,255,0.06620266536871593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,1023,0.13124799728393555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,255,0.06755200028419495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,1023,0.1912213365236918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,1,0.022096000611782074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,1,0.02367466688156128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,float16,511,0.10971732934315999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,3,0.02237333357334137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,40,8,128,1,float16,fp8,511,0.08719999591509502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,3,0.023397333920001984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,7,0.02201066662867864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,7,0.02348266790310542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,15,0.023717333873112995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,15,0.0220266655087471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,31,0.02404800057411194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,63,0.02346133440732956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,31,0.02201066662867864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,127,0.022687998910744984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,127,0.02605333427588145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,63,0.022069332500298817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,float16,255,0.029834667841593426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,1,128,1,float16,fp8,255,0.028069332242012024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,1,0.028736000259717304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,1,0.029834667841593426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,3,0.02844800055027008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,3,0.03029866764942805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,7,0.028037334481875103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,7,0.030063999195893604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,15,0.028933333853880566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,15,0.03009066730737686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,31,0.03002133220434189
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,31,0.02905600021282832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,63,0.02871999889612198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,63,0.030389333764712017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,127,0.032586666444937386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,127,0.03456533451875051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,float16,255,0.049072002371152244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,1,0.04128533353408178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,2,128,1,float16,fp8,255,0.0423573354880015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,1,0.044826666514078774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,3,0.041402667760849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,7,0.0408693328499794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,3,0.04460266729195913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,7,0.04410133262475332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,15,0.04204266766707102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,15,0.04409066836039225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,31,0.0415040006240209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,31,0.044810667634010315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,63,0.044997334480285645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,63,0.04428799947102865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,127,0.051370665431022644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,127,0.05629866818586985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,1,0.06685866912206014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,fp8,255,0.06783999999364217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,4,128,1,float16,float16,255,0.07021866738796234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,1,0.07191466788450877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,7,0.0673333356777827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,3,0.06685866912206014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,3,0.07126933336257935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,7,0.07119999825954437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,15,0.06724800169467926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,15,0.0713919997215271
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,31,0.07217599948247273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,31,0.07212266822655995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,63,0.0739519993464152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,63,0.0755573312441508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,127,0.0802293320496877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,127,0.09340266386667888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,float16,255,0.11714133620262146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,40,8,128,1,float16,fp8,255,0.11334932843844096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,1,0.04081066697835922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,1,0.04214933514595032
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,3,0.04029333343108495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,3,0.041802664597829185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,7,0.04062933226426443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,7,0.04188266893227895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,15,0.042538667718569435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,31,0.04081599911053976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,15,0.0408746674656868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,31,0.041893333196640015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,63,0.04127466678619385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,63,0.042037333051363625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,float16,127,0.04837866624196371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,1,128,1,float16,fp8,127,0.046240001916885376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,1,0.055120001236597695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,1,0.05292266607284546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,3,0.05276266733805338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,3,0.05467733244101206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,7,0.05259199937184652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,7,0.055231998364130654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,15,0.05241066714127859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,15,0.0547626664241155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,31,0.05320000151793162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,31,0.05522133409976959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,63,0.05514666438102722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,63,0.05972266693909963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,float16,127,0.06617600222428639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,2,128,1,float16,fp8,127,0.06810666620731354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,1,0.07829866806666057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,1,0.08334933718045552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,3,0.07904000083605449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,3,0.08409600456555684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,7,0.07806933422883351
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,7,0.08409600456555684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,15,0.07962666451931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,15,0.08285866677761078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,31,0.08635200063387553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,31,0.08370133241017659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,63,0.08780800302823384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,63,0.09058133761088054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,float16,127,0.09409067034721375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,1,0.13051733374595642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,4,128,1,float16,fp8,127,0.1081813375155131
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,1,0.1355839967727661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,3,0.12997866670290628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,3,0.1360106666882833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,7,0.13156800468762717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,15,0.1376213332017263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,7,0.13739200433095297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,15,0.13743467132250467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,31,0.1402666668097178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,31,0.14313600460688272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,63,0.14075733224550882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,63,0.1442400018374125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,1,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,3,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,3,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,7,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,15,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,7,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,15,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,31,0.01146666705608368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,float16,127,0.15172266960144043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,31,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,63,0.011381333072980246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,40,8,128,1,float16,fp8,127,0.17684799432754517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,63,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,127,0.011071999867757162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,127,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,255,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,511,0.013855999956528345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,511,0.015466666469971338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,255,0.014352000008026758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,1023,0.017968000223239262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,2047,0.020128000527620316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,1023,0.016000000139077503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,2047,0.022511998812357586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,4095,0.02906133234500885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,4095,0.03198933353026708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,8191,0.04760533571243286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,8191,0.05212266743183136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,float16,16383,0.08406933148701985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,1,128,1,float16,fp8,16383,0.09540266791979472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1,0.011621333658695221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,15,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,1,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,3,0.011744000017642975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,3,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,7,0.01166933278242747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,7,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,15,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,31,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,31,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,63,0.011605333536863327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,63,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,127,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,127,0.013376000026861826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,255,0.012634667257467905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,255,0.015743999431530636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,511,0.014901333798964819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,511,0.01600533351302147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,4095,0.02978666623433431
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,1023,0.01642666632930438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,1023,0.01844266677896182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,2047,0.02062400057911873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,2047,0.023013333479563396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,4095,0.033226666351159416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,8191,0.04877333343029022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,8191,0.0551093320051829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,float16,16383,0.08405866225560506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,3,0.012879999975363413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,2,128,1,float16,fp8,16383,0.1002293328444163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1,0.011706666400035223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,1,0.012666666259368261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,3,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,7,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,7,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,15,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,15,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,31,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,31,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,63,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,63,0.013141332815090815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,127,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,127,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,255,0.013744000345468521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,255,0.01545599972208341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,511,0.015194666882356008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,511,0.016165333489576977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,1023,0.016554666062196095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,1023,0.01894933357834816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,2047,0.021557333568731945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,2047,0.023413332800070446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,4095,0.03791466603676478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,4095,0.0346666673819224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,8191,0.07098133365313213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,8191,0.058245331048965454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,float16,16383,0.12834133704503378
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,4,128,1,float16,fp8,16383,0.10098133484522502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,1,0.012725333372751871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,15,0.013365333278973898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,3,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,3,0.012842666357755661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,7,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,7,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,15,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,31,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,31,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,63,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,63,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,127,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,127,0.014533333480358124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,1023,0.018837332725524902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,255,0.014058666924635569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,255,0.015813333292802174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,511,0.015520000209410986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,511,0.017008000363906223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,1023,0.018826667219400406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,2047,0.03659199923276901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,2047,0.025120000044504803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,16383,0.18478399515151978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,4095,0.057861333092053734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,4095,0.04191466669241587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,float16,8191,0.10069333513577779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,8191,0.06459199885527293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,40,8,128,1,float16,fp8,16383,0.12683199842770895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,1,0.03793066740036011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,1,0.03832533210515976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,3,0.037845333417256675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,3,0.038762666285037994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,7,0.03812800099452337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,7,0.038319999972979225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,15,0.03881066789229711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,15,0.03873066604137421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,31,0.043290664752324425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,31,0.03853866706291834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,float16,63,0.04604266583919525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,1,128,1,float16,fp8,63,0.04089066634575526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,1,0.06233066817124685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,3,0.0644160012404124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,3,0.06310933331648509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,7,0.06516266862551372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,7,0.06268799801667531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,15,0.06865066786607106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,15,0.0636053333679835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,31,0.07194666564464569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,31,0.06463466584682465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,1,0.06426133215427399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,float16,63,0.07434666653474171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,2,128,1,float16,fp8,63,0.07277866701285045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,1,0.11499733726183574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,1,0.11204266548156738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,3,0.11753599842389424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,3,0.11317333579063416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,7,0.12429333726565044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,7,0.11355200409889221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,15,0.12769066294034323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,15,0.1170186698436737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,31,0.1276693344116211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,31,0.12597866853078207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,float16,63,0.12997866670290628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,4,128,1,float16,fp8,63,0.12749333182970682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,1,0.2111253341039022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,1,0.20665599902470908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,3,0.22218134005864462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,3,0.20548800627390543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,7,0.22813334067662558
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,7,0.21502399444580078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,15,0.23097066084543863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,15,0.22313600778579712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,31,0.22987733284632364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,float16,63,0.23327465852101645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,31,0.224346657594045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,40,8,128,1,float16,fp8,63,0.2251680095990499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,1,0.07649066547552745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,1,0.06997333467006683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,3,0.07040533423423767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,7,0.07565333445866902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,7,0.07206933200359344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,15,0.07351466516653697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,15,0.07701866825421651
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,3,0.07764799892902374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,float16,31,0.07600000003973643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,1,128,1,float16,fp8,31,0.07869866490364075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,1,0.13081600268681845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,1,0.11668266852696736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,3,0.12985600034395853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,3,0.12086400389671326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,7,0.12270933389663696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,15,0.13056000073750815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,15,0.1279039978981018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,31,0.1321386694908142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,fp8,31,0.12772267063458762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,2,128,1,float16,float16,7,0.13177067041397095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,1,0.23925334215164185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,1,0.22620266675949097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,3,0.24105066061019897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,3,0.2286720077196757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,7,0.2434719999631246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,7,0.2326293389002482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,15,0.24008532365163168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,float16,31,0.2421600023905436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,15,0.2341653307278951
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,4,128,1,float16,fp8,31,0.23483733336130777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,1,0.4455626805623372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,1,0.4214773178100586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,3,0.44916268189748126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,3,0.42863468329111737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,7,0.44334399700164795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,1,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,7,0.42424531777699787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,3,0.011690666278203329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,7,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,3,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,7,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,15,0.011711999773979187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,31,0.011706666400035223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,15,0.4280159870783488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,15,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,31,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,15,0.4503093163172404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,63,0.011744000017642975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,63,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,127,0.011674666156371435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,127,0.013450667262077332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,255,0.013141332815090815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,511,0.014954666296641031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,255,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,511,0.016757333030303318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,1023,0.01924266666173935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,float16,31,0.44836799303690594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,40,8,128,1,float16,fp8,31,0.4269760052363078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,1023,0.021562665700912476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,2047,0.028223998844623566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,2047,0.031354665756225586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,4095,0.047151997685432434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,4095,0.05013333261013031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,float16,8191,0.08261333405971527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,1,0.013189333180586496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,1,128,1,float16,fp8,8191,0.0938933293024699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,3,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,3,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,7,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,63,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,7,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,15,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,15,0.013295999417702356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,31,0.01227733368674914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,255,0.013621332744757334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,31,0.012789333860079447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,63,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,127,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,1023,0.02012266715367635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,1023,0.022405333817005157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,127,0.014592000593741735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,255,0.01482133318980535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,511,0.01553600033124288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,511,0.01748266691962878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,2047,0.029498666524887085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,8191,0.09922666351000468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,2047,0.03214933226505915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,4095,0.04982399940490723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,3,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,fp8,4095,0.053120002150535583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,7,0.015775999675194424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,2,128,1,float16,float16,8191,0.08522666494051616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,1,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,3,0.013104000439246496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,7,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,15,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,15,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,31,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,31,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,63,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,63,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,127,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,127,0.014346666634082794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,255,0.014058666924635569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,255,0.014933332800865173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,511,0.01617066686352094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,511,0.017781333376963932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,1023,0.02117866774400075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,1023,0.02199999988079071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,2047,0.036837334434191384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,2047,0.033520000676314034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,4095,0.0582239975531896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,4095,0.05658133327960968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,3,0.013295999417702356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,float16,8191,0.10202133655548096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,4,128,1,float16,fp8,8191,0.09891200065612793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,1,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,3,0.012421333541472753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,7,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,7,0.013013333082199097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,15,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,15,0.013717333475748697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,31,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,255,0.014576000471909841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,31,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,63,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,63,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,127,0.012810666114091873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,127,0.014671999961137772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,255,0.015615999698638916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,511,0.016864000509182613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,511,0.018079999834299088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,1023,0.0348693331082662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,1023,0.025957333544890087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,2047,0.05584000051021576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,2047,0.04016000032424927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,4095,0.0983893374602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,4095,0.06366933385531108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,float16,8191,0.18346667289733887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,40,8,128,1,float16,fp8,8191,0.10935999949773152
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,1,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,3,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,3,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,7,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,7,0.012938667088747025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,15,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,15,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,31,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,31,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,63,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,63,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,127,0.012565333396196365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,127,0.014384000251690546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,1023,0.021695998807748158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,255,0.01358933374285698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,255,0.014815999815861383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,511,0.015344000111023584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,511,0.017114666601022083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,1023,0.01969066634774208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,2047,0.02995733420054118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,2047,0.0322026660044988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,4095,0.04967466493447622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,4095,0.053445334235827126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,3,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,float16,8191,0.08565333485603333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,1,128,1,float16,fp8,8191,0.10017599662144978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,1,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,3,0.013157332936922709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,7,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,7,0.013173333058754602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,15,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,15,0.01321600005030632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,31,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,31,0.012826666235923767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,63,0.012469333906968435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,63,0.013082666943470636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,127,0.012703999876976013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,127,0.014416000495354334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,255,0.014256000518798828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,255,0.014975999792416891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,511,0.015775999675194424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,511,0.01815466706951459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,1023,0.02120000123977661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,1023,0.02232533444960912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,8191,0.10180800159772237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,2047,0.03658666710058848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,2047,0.03315199911594391
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,float16,4095,0.05879466732343038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,4095,0.05775466561317444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,2,128,1,float16,fp8,8191,0.10054399569829305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,3,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,1,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,3,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,7,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,7,0.013376000026861826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,15,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,15,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,31,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,31,0.013173333058754602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,63,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,63,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,127,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,127,0.014885333677132925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,255,0.014303999642531076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,255,0.015498666713635126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,511,0.01754133279124896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,511,0.018229333062966663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,1023,0.03504000107447306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,1023,0.0233599990606308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,8191,0.18251200517018637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,2047,0.05625600119431814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,2047,0.03975466638803482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,float16,4095,0.0979200005531311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,4095,0.0635040005048116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,4,128,1,float16,fp8,8191,0.10885866483052571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1,0.016330666840076447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,1,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,3,0.016480000068744022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,3,0.017456000049908955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,7,0.01669866715868314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,7,0.017802666872739792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,15,0.016597333053747814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,15,0.017407999684413273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,31,0.016538667182127636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,31,0.017690667261679966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,255,0.020992000897725422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,63,0.016629333297411602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,63,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,511,0.03009066730737686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,127,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,127,0.020448000480731327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,255,0.021962667504946392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,2047,0.06776000062624614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,511,0.03755733370780945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,1023,0.05830933153629303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,1023,0.04312533140182495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,2047,0.09963732957839966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,4095,0.18448533614476523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,4095,0.1132586697737376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,float16,8191,0.3531893491744995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,32,8,128,1,float16,fp8,8191,0.20404799779256186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,15,0.011045332998037338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,1,0.012453333785136541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,3,0.010960000256697336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,3,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,7,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,7,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,15,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,31,0.011215999722480774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,31,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,255,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,63,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,511,0.01350933313369751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,63,0.01249066616098086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,127,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,127,0.013157332936922709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,255,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,511,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,4095,0.01626666635274887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,8191,0.01740266631046931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,1023,0.012800000607967377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,16383,0.022272000710169475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,1023,0.014293332894643148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,2047,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,2047,0.015135999768972397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,4095,0.01505600040157636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,8191,0.018906666586796444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,16383,0.024325333535671234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,float16,32767,0.03169066707293192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,1,128,1,float16,fp8,32767,0.03453333427508672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,1,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,3,0.010826667149861654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,3,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,7,0.01089599976936976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,7,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,15,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,15,0.012698666503032049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,31,0.011018666128317514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,31,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,255,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,63,0.010970667004585266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,63,0.012304000556468964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,127,0.011168000598748526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,127,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,1023,0.01440000037352244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,255,0.013301332791646322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,511,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,511,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,1023,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,2047,0.013989333063364029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,2047,0.015599999576807022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,4095,0.015925332903862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,4095,0.016634666671355564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,32767,0.035162667433420815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,8191,0.018266666680574417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,8191,0.019434666881958645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,16383,0.023354666928450268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,fp8,16383,0.024874667326609295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,2,128,1,float16,float16,32767,0.03369600077470144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,1,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,3,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,3,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,7,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,7,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,15,0.011498666057984034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,15,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,31,0.011168000598748526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,31,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,255,0.012885333349307379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,63,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,63,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,127,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,127,0.01328533391157786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,2047,0.015072000523408255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,255,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,511,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,511,0.013872000078360239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,1023,0.014218666901191076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,1023,0.015029333531856537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,2047,0.015802666544914246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,4095,0.01685333376129468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,4095,0.01732800031701724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,8191,0.018719999740521114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,8191,0.01929066702723503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,16383,0.023130667706330616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,16383,0.025311999022960663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,float16,32767,0.03775999943415324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,4,128,1,float16,fp8,32767,0.03547733277082443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,1,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,3,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,31,0.011472000430027643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,3,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,7,0.011472000430027643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,7,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,15,0.011488000551859537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,15,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,31,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,63,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,63,0.012469333906968435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,511,0.014373333503802618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,1023,0.013418667018413544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,127,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,127,0.013050666699806849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,255,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,255,0.014106666048367819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,511,0.013376000026861826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,8191,0.02065066620707512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,1023,0.015557333827018738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,2047,0.015013333410024643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,2047,0.016127999871969223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,4095,0.016864000509182613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,4095,0.01746133342385292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,8191,0.02040533348917961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,16383,0.03737599899371465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,3,0.01091733326514562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,16383,0.026928000152111053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,float16,32767,0.05961066484451294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,7,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,7,0.010837333897749582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,32,8,128,1,float16,fp8,32767,0.0465280016263326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1,0.011050666371981302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,31,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,1,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,3,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,15,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,31,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,15,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,63,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,63,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,511,0.012815999488035837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,127,0.010954666882753372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,127,0.01332266628742218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,1023,0.01523200049996376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,255,0.012373333175977072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,255,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,4095,0.017610666652520496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,511,0.01379199946920077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,1023,0.013530666629473368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,8191,0.0234400009115537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,2047,0.014933332800865173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,2047,0.01594666639963786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,4095,0.01842133328318596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,8191,0.020794666061798733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1,0.01138666644692421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,16383,0.028938665986061096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,16383,0.0331839993596077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,float16,32767,0.04722133278846741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,7,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,1,128,1,float16,fp8,32767,0.05243733525276184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,1,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,3,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,3,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,7,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,15,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,15,0.012645332763592402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,31,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,31,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,63,0.01146666705608368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,511,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,63,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,127,0.011482667177915573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,127,0.013376000026861826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,255,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,2047,0.01651200031240781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,255,0.013669333110253016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,511,0.014458666245142618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,8191,0.02165333429972331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,1023,0.014170666535695394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,1023,0.01589866727590561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,2047,0.016037333756685257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,4095,0.018415999909241993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,4095,0.019178666174411774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,8191,0.02382933348417282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,16383,0.030949334303538006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,16383,0.033600000043710075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,float16,32767,0.05086933573087057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,2,128,1,float16,fp8,32767,0.05429866909980774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1,0.011413333316644033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,1,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,3,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,31,0.011493333925803503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,3,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,63,0.011482667177915573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,7,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,7,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,15,0.011488000551859537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,15,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,31,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,511,0.01393066719174385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,63,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,127,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,127,0.013194666554530462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,255,0.01258133351802826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,255,0.013722666849692663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,511,0.014208000153303146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,1023,0.014922666052977243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,1023,0.016021333634853363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,2047,0.0161920003592968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,2047,0.01655999943614006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,4095,0.0183146670460701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,4095,0.019487999379634857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,8191,0.021962667504946392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,8191,0.023760000864664715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,16383,0.037274666130542755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,16383,0.03391999999682108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,3,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,float16,32767,0.06181333462397257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,4,128,1,float16,fp8,32767,0.056661332647005715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1,0.011434666812419891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,15,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,1,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,3,0.011519999553759893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,7,0.011802667131026586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,7,0.012608000387748083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,15,0.011482667177915573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,31,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,31,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,63,0.011701333026091257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,63,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,511,0.014709333578745524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,127,0.011626667032639185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,127,0.014186666657527288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,255,0.013338666409254074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,255,0.014240000396966934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,2047,0.01699200024207433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,511,0.014085333794355392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,1023,0.01488000030318896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,1023,0.015967999895413715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,2047,0.016677333662907284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,4095,0.019519999623298645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,4095,0.019685332973798115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,8191,0.036848001182079315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,8191,0.0271519993742307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,16383,0.05792533357938131
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,16383,0.04144000013669332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,float16,32767,0.09988799691200256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,32,8,128,1,float16,fp8,32767,0.07308266560236613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1,0.012624000509579977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,1,0.013354666531085968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,3,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,3,0.013482666263977686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,7,0.012752000242471695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,31,0.013418667018413544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,7,0.013471999516089758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,15,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,15,0.013237333546082178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,31,0.012565333396196365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,63,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,63,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,127,0.01357866699496905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,127,0.015125333021084467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,255,0.01471466695268949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,255,0.015664000064134598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,2047,0.03724266588687897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,511,0.01635733370979627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,511,0.018101333330074947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,1023,0.022128000855445862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,4095,0.05825600028038025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,1023,0.022853332261244457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,fp8,2047,0.03365333378314972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,1,128,1,float16,float16,4095,0.059893334905306496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,1,0.014117332796255747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,3,0.012906666845083237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,3,0.014463999619086584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,7,0.014165333161751429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,7,0.01314666618903478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,15,0.01333333303531011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,31,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,15,0.013989333063364029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,31,0.014170666535695394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,63,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,63,0.014581333845853806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,127,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,127,0.015615999698638916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,255,0.015200000256299973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,255,0.01618133361140887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,511,0.018101333330074947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,511,0.018895999838908512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,1023,0.0356480007370313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,1023,0.025914666553338368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,2047,0.056885331869125366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,2047,0.04227200150489807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,float16,4095,0.09889066219329834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,3,0.01674666628241539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,2,128,1,float16,fp8,4095,0.06400533517201741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1,0.016688000410795212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,1,0.018351999421914417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,3,0.01806933308641116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,7,0.016805333395799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,15,0.016970666746298473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,63,0.01693333312869072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,7,0.017792000124851864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,15,0.017690667261679966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,127,0.020687999824682873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,31,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,31,0.017610666652520496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,63,0.018186666071414948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,127,0.01727466657757759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,255,0.0216799999276797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,255,0.022096000611782074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,511,0.03722666700681051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,511,0.028565332293510437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,1023,0.059530665477116905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,4095,0.1841813325881958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,1023,0.044581333796183266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,float16,2047,0.10124799609184265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,2047,0.06789866586526234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,4,128,1,float16,fp8,4095,0.11270933349927266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1,0.023589332898457844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,1,0.025487999121348064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,3,0.02359466751416524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,3,0.02517866591612498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,7,0.023813332120577495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,7,0.025536000728607178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,15,0.02407466620206833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,15,0.025285333395004272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,31,0.023872000475724537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,31,0.02554133286078771
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,63,0.024165332317352295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,63,0.02533866713444392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,255,0.03405333310365677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,127,0.025125332176685333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,127,0.030389333764712017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,255,0.042266666889190674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,511,0.06259733438491821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,511,0.05106666684150696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,1023,0.10259733597437541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,1023,0.07322133580843608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,2047,0.18506133556365967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,2047,0.12165866295496623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,float16,4095,0.3531839847564697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1,0.011215999722480774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,1,0.011685332904259363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,32,8,128,1,float16,fp8,4095,0.2099519968032837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,3,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,3,0.01137599969903628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,7,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,7,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,15,0.010970667004585266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,127,0.011045332998037338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,15,0.011530666301647821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,31,0.010762666662534079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,31,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,63,0.011039999624093374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,511,0.014293332894643148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,63,0.011626667032639185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,127,0.012991999586423239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,2047,0.015754666179418564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,255,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,255,0.01351999988158544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,511,0.013642666240533194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,1023,0.014645333091417948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,1023,0.015498666713635126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,2047,0.017893332988023758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,4095,0.02014933278163274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,4095,0.022458667556444805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,8191,0.02889599899450938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,8191,0.032746667663256325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,16383,0.047226667404174805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,16383,0.05136000116666158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,float16,32767,0.08272533118724823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,1,128,1,float16,fp8,32767,0.09181867043177287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1,0.011178666104873022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,15,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,1,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,3,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,3,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,7,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,7,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,15,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,31,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,31,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,255,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,63,0.011045332998037338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,63,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,127,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,127,0.01313599944114685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,255,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,511,0.013642666240533194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,4095,0.021125334004561108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,4095,0.023071999351183575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,511,0.015498666713635126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,8191,0.029637334247430164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,1023,0.01571200042963028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,1023,0.01591466615597407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,2047,0.016719999412695568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,2047,0.018768000106016796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,8191,0.03350399931271871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,16383,0.05029866596062978
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,16383,0.054048001766204834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,3,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,float16,32767,0.08375466863314311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,2,128,1,float16,fp8,32767,0.10016000270843506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,1,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,3,0.011600000162919363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,7,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,7,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,15,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,127,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,15,0.011786667009194693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,31,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,31,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,63,0.011503999431928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,63,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,127,0.013573333621025085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,255,0.013546666751305262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,255,0.01368533323208491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,511,0.01443733274936676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,511,0.015087999403476715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,1023,0.01562133307258288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,1023,0.015935999651749928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,8191,0.035114665826161705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,2047,0.016986666868130367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,16383,0.05492266515890757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,2047,0.018570666511853535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,4095,0.021701333423455555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,4095,0.023200000325838726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,8191,0.0373279998699824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,16383,0.06155199805895487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,float16,32767,0.109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,4,128,1,float16,fp8,32767,0.095551997423172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1,0.011701333026091257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,1,0.012773333738247553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,15,0.011781333635250727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,3,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,3,0.012719999998807907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,7,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,7,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,15,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,127,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,31,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,31,0.012960000584522883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,63,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,63,0.012789333860079447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,127,0.014122666170199713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,255,0.01314666618903478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,255,0.014058666924635569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,511,0.014741333822409311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,511,0.016106666376193363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,1023,0.01643199970324834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,1023,0.01642666632930438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,2047,0.0183146670460701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,2047,0.019082666685183842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,4095,0.036789332826932274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,4095,0.02366400013367335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,8191,0.05764266848564148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,8191,0.045466666420300804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,16383,0.10012267033259074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,16383,0.06391466657320659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,float16,32767,0.1851093371709188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,32,8,128,1,float16,fp8,32767,0.11058132847150166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,3,0.014554666976133982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1,0.014346666634082794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,1,0.015674666812022526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,3,0.015775999675194424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,7,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,31,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,7,0.015615999698638916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,15,0.014389333625634512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,15,0.015493333339691162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,31,0.01552533358335495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,127,0.017279999951521557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,63,0.014362666755914688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,63,0.015893333901961643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,127,0.014607999473810196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,255,0.01669866715868314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,255,0.01738133281469345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,511,0.019386666516462963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,511,0.020373333245515823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,1023,0.03682666768630346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,2047,0.04332800209522247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,fp8,1023,0.026330667237440746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,1,128,1,float16,float16,2047,0.05868266522884369
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1,0.01764800027012825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,1,0.018986667195955913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,3,0.017722666263580322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,3,0.019007999449968338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,7,0.017850667238235474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,15,0.01937599976857503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,7,0.018981333822011948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,15,0.017786666750907898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,31,0.017770666629076004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,31,0.019023999571800232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,63,0.017770666629076004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,255,0.022416000564893086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,63,0.019093333433071773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,127,0.018506667266289394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,127,0.021925332645575207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,511,0.029717333614826202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,255,0.023226665953795116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,511,0.03990933299064636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,1023,0.0622026671965917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,1023,0.046666666865348816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1,0.024277334411938984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,1,0.02569066733121872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,3,0.023999998966852825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,float16,2047,0.10291733344395955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,3,0.025909334421157837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,2,128,1,float16,fp8,2047,0.07190933326880138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,7,0.02402666707833608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,7,0.025786665578683216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,15,0.024522667129834492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,15,0.025573333104451496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,31,0.025722667574882507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,31,0.025072000920772552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,63,0.024874667326609295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,127,0.026799999177455902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,127,0.030810666580994923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,63,0.02608533451954524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,255,0.04186666508515676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,255,0.03549866626660029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,511,0.0631573349237442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,2047,0.12092266480127971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,2047,0.18638400236765543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,511,0.05192000170548757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1,0.03800000001986822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,1,0.039919999738534294
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,fp8,1023,0.07430399954319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,3,0.04065600037574768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,7,0.03700799991687139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,3,0.03728000074625015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,4,128,1,float16,float16,1023,0.10381333033243816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,15,0.040005333721637726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,7,0.040133332212766014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,15,0.038176000118255615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,31,0.04027199993530909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,31,0.037978666524092354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,63,0.03979199876387914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,127,0.04618666569391886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,127,0.05134400228659312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,63,0.04092800120512644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,255,0.06844800213972728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,255,0.06235733131567637
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,2047,0.3552639881769816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,1023,0.13170666495958963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,1023,0.19155732790629068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,2047,0.22581867376963297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,float16,511,0.10986666878064473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,32,8,128,1,float16,fp8,511,0.08725866675376892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,1,0.023381332556406658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,1,0.02160000056028366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,3,0.02178666740655899
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,3,0.023200000325838726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,7,0.021695998807748158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,7,0.023408000667889912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,15,0.021477334201335907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,31,0.02163200080394745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,31,0.02332799881696701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,15,0.023077333966890972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,63,0.022133332987626392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,127,0.0223786657055219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,63,0.023333333432674408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,255,0.028581333657105763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,127,0.02619733413060506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,255,0.02773333340883255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,float16,511,0.04491200049718221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,1,128,1,float16,fp8,511,0.03610666592915853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,1,0.02979733298222224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,1,0.028160000840822857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,3,0.027877333263556164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,3,0.029978667696317036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,7,0.028202667832374573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,7,0.03012799968322118
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,15,0.027999999622503918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,15,0.02978666623433431
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,31,0.030133334298928578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,31,0.02815466622511546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,63,0.028234665592511494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,63,0.030026666820049286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,127,0.03164266546567281
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,127,0.0347626656293869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,255,0.04680533210436503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,255,0.0418453315893809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,float16,511,0.06993600229422252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,2,128,1,float16,fp8,511,0.05715199808279673
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,1,0.04052799940109253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,1,0.04282666742801666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,3,0.040762667854626976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,3,0.043050666650136314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,7,0.04004266609748205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,7,0.042597333590189614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,15,0.041434665520985924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,15,0.04252799848715464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,31,0.04084266722202301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,31,0.04287466903527578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,63,0.04346133271853129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,63,0.043007999658584595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,127,0.04960533479849497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,127,0.05514666438102722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,255,0.07025066514809926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,255,0.06596266726652782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,1,0.06712000072002411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,float16,511,0.11106133460998535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,1,0.07141333321730296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,4,128,1,float16,fp8,511,0.08960533142089844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,3,0.06692799925804138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,3,0.07073066631952922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,7,0.06713599960009257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,7,0.07037333150704701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,15,0.06727999945481618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,31,0.07030400137106578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,63,0.07486933469772339
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,15,0.07127466797828674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,63,0.07160000006357829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,127,0.08084266881148021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,31,0.07101333141326904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,127,0.09305066863695781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,1,0.03886933376391729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,255,0.11799466609954834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,255,0.10900800426801045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,3,0.03885866701602936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,1,0.04119999955097834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,3,0.04088533421357473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,float16,511,0.20279467105865479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,32,8,128,1,float16,fp8,511,0.15611199537913004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,7,0.03861333429813385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,15,0.04111466556787491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,7,0.04195199906826019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,31,0.038912000755469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,31,0.04102933406829834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,15,0.03896533449490865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,127,0.044863998889923096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,63,0.03839466720819473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,127,0.045269335309664406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,63,0.04071466624736786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,float16,255,0.05886933207511902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,1,0.05111999809741974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,1,128,1,float16,fp8,255,0.055029332637786865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,1,0.05372266471385956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,3,0.05106666684150696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,3,0.05372266471385956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,7,0.050586665670077004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,7,0.05412266651789347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,15,0.05132266879081726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,15,0.05352533360322317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,31,0.05089066425959269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,63,0.056501333912213646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,31,0.05444266895453135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,63,0.054042667150497437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,127,0.06326400240262349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,127,0.06782933572928111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,float16,255,0.08261333405971527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,1,0.0751039981842041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,2,128,1,float16,fp8,255,0.07939200103282928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,1,0.08012266457080841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,3,0.07695466776688893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,3,0.08040533463160197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,7,0.07555200159549713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,7,0.08115200201670329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,15,0.07760000228881836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,15,0.07965333263079326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,31,0.08080000181992848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,31,0.08046400050322215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,63,0.08418666323026021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,127,0.08928533395131429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,63,0.08538132905960083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,127,0.10427199800809224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,float16,255,0.12442666292190552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,1,0.1274133324623108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,4,128,1,float16,fp8,255,0.11780266960461934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,1,0.13382933537165323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,3,0.13521066308021545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,3,0.12956266601880392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,7,0.13715733091036478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,7,0.1291039983431498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,15,0.1332266628742218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,31,0.133733332157135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,15,0.1346879998842875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,31,0.1416213313738505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,63,0.13729600111643472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,63,0.14333867033322653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,127,0.1497653325398763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,127,0.17456533511479697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,3,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,float16,255,0.2198773423830668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,1,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,15,0.011071999867757162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,3,0.011247999966144562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,7,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,7,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,15,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,32,8,128,1,float16,fp8,255,0.20690133174260458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,127,0.01333333303531011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,31,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,63,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,31,0.011114666859308878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,63,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,127,0.010944000134865442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,255,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,511,0.013839999834696451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,511,0.015125333021084467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,255,0.014256000518798828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,1023,0.015743999431530636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,1023,0.017866666118303936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,2047,0.020090666910012562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,16383,0.0832479993502299
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,2047,0.02233600119749705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,4095,0.03035733352104823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,4095,0.03238933285077413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,8191,0.04712533454100291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,8191,0.05123733480771383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,16383,0.09637866417566936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,float16,32767,0.15197333693504333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,1,128,1,float16,fp8,32767,0.1840320030848185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,7,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1,0.011514666179815928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,1,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,3,0.011733333269755045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,31,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,63,0.011706666400035223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,3,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,7,0.011493333925803503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,15,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,15,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,31,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,63,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,127,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,127,0.013525333255529404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,255,0.012703999876976013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,255,0.015647999942302704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,511,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,2047,0.02294933299223582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,511,0.015824000040690105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,1023,0.01603200038274129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,1023,0.018394666413466137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,2047,0.021205333371957142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,4095,0.030640001098314922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,4095,0.03257599969704946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,8191,0.05075199902057648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,8191,0.055402666330337524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,16383,0.08405333757400513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,16383,0.10152000188827515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,float16,32767,0.15737066666285196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,2,128,1,float16,fp8,32767,0.18148267269134521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,7,0.012853333105643591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1,0.011648000528415045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,1,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,3,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,3,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,63,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,7,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,15,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,15,0.012517333030700684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,31,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,255,0.015685333559910457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,31,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,63,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,127,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,1023,0.018144000321626663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,127,0.014469332993030548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,255,0.014069333672523499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,511,0.014602666099866232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,511,0.016176000237464905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,1023,0.016751999656359356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,8191,0.05479466418425242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,2047,0.020986666282018025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,2047,0.023306667804718018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,4095,0.03608533243338267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,32767,0.17124799887339273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,4095,0.033914667864640556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,8191,0.05987200140953064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,16383,0.12693867087364197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,fp8,16383,0.09515733520189922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,4,128,1,float16,float16,32767,0.20120000839233398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,7,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,1,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,31,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,3,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,3,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,63,0.012938667088747025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,7,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,15,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,15,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,31,0.01309866706530253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,63,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,127,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,127,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,255,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,255,0.015850666910409927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,511,0.015397333850463232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,511,0.01700266698996226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,4095,0.04132800052563349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,1023,0.018250666558742523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,1023,0.018794666975736618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,2047,0.03696000079313914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,2047,0.025263999899228413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,4095,0.05831466615200043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,8191,0.0997759997844696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,8191,0.06491733094056447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,16383,0.18473066886266074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,16383,0.12690132856369019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,float16,32767,0.3551679849624634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,32,8,128,1,float16,fp8,32767,0.22817067305246988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,1,0.03788266579310099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,1,0.03849066545565923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,3,0.038202665746212006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,3,0.03827733298142751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,7,0.038005332152048744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,7,0.0384853333234787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,15,0.03807999938726425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,15,0.03825599948565165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,31,0.03986666599909464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,31,0.03885866701602936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,63,0.045279999574025474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,float16,127,0.04926399886608124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,63,0.04061333338419596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,1,128,1,float16,fp8,127,0.05387733379999796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,1,0.06361599763234456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,1,0.062128002444903054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,3,0.0637600024541219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,3,0.06298133234182994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,7,0.06364266574382782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,7,0.06274133423964183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,15,0.06678933401902516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,15,0.06311999758084615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,31,0.07106133302052815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,31,0.06357333560784657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,63,0.07332266867160797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,63,0.07253866891066234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,float16,127,0.0783786674340566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,2,128,1,float16,fp8,127,0.08830933769543965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,1,0.111135999361674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,1,0.11127466956774394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,3,0.10982400178909302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,3,0.11125333110491435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,7,0.1118239959081014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,7,0.11045333743095398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,15,0.11896533767382304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,31,0.12127466996510823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,15,0.11261333028475444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,31,0.11863999565442403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,63,0.12377599875132243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,63,0.12262399991353352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,float16,127,0.13685333728790283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,1,0.21214399735132852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,4,128,1,float16,fp8,127,0.15221866965293884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,1,0.2050186594327291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,3,0.20385066668192545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,3,0.21876267592112222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,7,0.22642666101455688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,7,0.21016534169514975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,15,0.2247520089149475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,31,0.22266666094462076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,31,0.22830400864283243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,15,0.2309866746266683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,63,0.2330346703529358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,63,0.23131199677785239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,float16,127,0.24770132700602213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,32,8,128,1,float16,fp8,127,0.2897439996401469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,1,0.07053333520889282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,1,0.06598400076230367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,3,0.07069333394368489
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,3,0.06654400130112965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,7,0.07332799832026164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,7,0.06738666693369548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,15,0.0747680018345515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,15,0.07053333520889282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,31,0.0759093314409256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,31,0.07502933343251546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,1,0.12422933181126912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,1,0.11482133467992146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,3,0.1271573305130005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,float16,63,0.0768746683994929
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,3,0.11474133531252544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,1,128,1,float16,fp8,63,0.07877866427103679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,7,0.12960533301035562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,7,0.11806933085123698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,15,0.12924266854921976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,15,0.12429866194725037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,31,0.12985600034395853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,31,0.12538133064905801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,float16,63,0.13212266564369202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,2,128,1,float16,fp8,63,0.1290720005830129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,1,0.21100266774495444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,1,0.21083199977874756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,3,0.21203200022379556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,7,0.22023999691009521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,3,0.21394133567810059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,7,0.2163626750310262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,15,0.22513065735499063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,15,0.22498667240142822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,31,0.23009600241978964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,float16,63,0.23357333739598593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,31,0.2262773315111796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,1,0.41257067521413165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,4,128,1,float16,fp8,63,0.2298346757888794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,1,0.4459093411763509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,3,0.44323734442392987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,3,0.42795201142628986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,7,0.4461013476053874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1,0.01156266654531161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,1,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,7,0.42275198300679523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,3,0.011424000064531961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,7,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,3,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,7,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,15,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,15,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,31,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,31,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,15,0.44434666633605957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,63,0.011616000284751257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,31,0.44308265050252277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,15,0.42370665073394775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,63,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,127,0.013482666263977686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,127,0.011701333026091257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,255,0.014287999520699183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,31,0.4362613360087077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,511,0.01693333312869072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,float16,63,0.45163734753926593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,1023,0.01930133377512296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,511,0.014938666174809137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,255,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,1023,0.02161066730817159
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,2047,0.031189332405726116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,2047,0.028186666468779247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,4095,0.046581332882245384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,32,8,128,1,float16,fp8,63,0.4333226680755615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,8191,0.08217066526412964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,8191,0.09090133508046468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,float16,16383,0.15227733055750528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,4095,0.05067199965318044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,1,128,1,float16,fp8,16383,0.18437333901723227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,1,0.012981332838535309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,3,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,7,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,7,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,15,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,3,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,15,0.012597333639860153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,31,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,31,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,63,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,63,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,255,0.015210667004187902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,127,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,127,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,255,0.013658666362365087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,511,0.015696000307798386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,511,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,1023,0.01989866668979327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,1023,0.022154666483402252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,2047,0.030042665700117748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,2047,0.032560000816980995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,4095,0.050255998969078064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,4095,0.05351466437180837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,8191,0.08243200182914734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,8191,0.09870933492978413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,3,0.012800000607967377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,float16,16383,0.15527466932932535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,2,128,1,float16,fp8,16383,0.18185067176818848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,15,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,1,0.013258667041858038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,3,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,31,0.012624000509579977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,7,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,63,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,7,0.012981332838535309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,15,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,31,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,511,0.015791999797026317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,63,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,127,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,127,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,255,0.014090667168299357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,255,0.015013333410024643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,511,0.01757866640885671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,1023,0.022015998760859173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,1023,0.022122666239738464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,2047,0.036602665980656944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,2047,0.03294933338960012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,4095,0.06073066592216492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,4095,0.0544053316116333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,8191,0.10714133580525716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,8191,0.09433600306510925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,float16,16383,0.20017067591349283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,4,128,1,float16,fp8,16383,0.17160000403722128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,1,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,3,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,3,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,7,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,7,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,15,0.012650666137536367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,15,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,31,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,31,0.013951999445756277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,63,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,63,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,127,0.013013333082199097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,127,0.014736000448465347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,255,0.014495999862750372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,255,0.015322666615247726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,511,0.016821333517630894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,511,0.018053332964579265
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,1023,0.03523733218510946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,1023,0.02351466566324234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,2047,0.055733333031336464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,2047,0.03997866561015447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,4095,0.0972106655438741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,4095,0.06278933087984721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,8191,0.18209065993626913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,8191,0.10896000266075134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,float16,16383,0.35229865709940594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,32,8,128,1,float16,fp8,16383,0.2000746726989746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,1,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,3,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,3,0.012304000556468964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,7,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,7,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,15,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,15,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,31,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,31,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,63,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,63,0.012896000097195307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,127,0.01249066616098086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,127,0.014101333916187286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,255,0.013925333817799887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,255,0.014954666296641031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,511,0.015594666202863058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,511,0.017221332838137943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,1023,0.01969066634774208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,1023,0.022005334496498108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,2047,0.02979733298222224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,2047,0.032032000521818794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,4095,0.04902400076389313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,4095,0.055018668373425804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,float16,8191,0.0830506682395935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,1,128,1,float16,fp8,8191,0.09987200299898784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,1,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,3,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,3,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,7,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,7,0.012768000364303589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,15,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,15,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,31,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,31,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,63,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,63,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,511,0.017551999539136887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,127,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,127,0.014287999520699183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,1023,0.021946666141351063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,255,0.014560000350077948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,255,0.014762666076421738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,4095,0.060362666845321655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,511,0.015461333096027374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,1023,0.022064000368118286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,8191,0.0990826686223348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,2047,0.037690666814645134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,3,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,2047,0.03482133398453394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,fp8,4095,0.05669866502285004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,2,128,1,float16,float16,8191,0.10673066973686218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,1,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,3,0.013471999516089758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,7,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,15,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,7,0.013466666142145792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,15,0.013658666362365087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,31,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,31,0.013440000514189402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,63,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,63,0.013642666240533194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,127,0.012826666235923767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,127,0.014650666465361914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,1023,0.034789333740870156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,255,0.014597332725922266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,255,0.015781333049138386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,511,0.017514667163292568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,511,0.018005333840847015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,1023,0.02402133246262868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,2047,0.056559999783833824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,2047,0.040336000422636666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,4095,0.09841600060462952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,4095,0.06277866661548615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,float16,8191,0.18337066968282065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,4,128,1,float16,fp8,8191,0.11052266756693523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1,0.016501333564519882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,1,0.01757866640885671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,3,0.016554666062196095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,3,0.017338667064905167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,7,0.016362667083740234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,7,0.01749333366751671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,15,0.01669866715868314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,15,0.017456000049908955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,31,0.016415999581416447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,31,0.017605333278576534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,63,0.016704000532627106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,63,0.017488000293572743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,127,0.016864000509182613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,127,0.020629333953062694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,255,0.021136000752449036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,255,0.02186133215824763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,511,0.03745600084463755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,511,0.027952000498771667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,1023,0.058415999015172325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,1023,0.04380266865094503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,2047,0.09928533434867859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,2047,0.06724266707897186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,1,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,4095,0.18395733833312988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,4095,0.11202667156855266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,7,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,float16,8191,0.3520266612370809
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,24,8,128,1,float16,fp8,8191,0.20373332500457764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,3,0.010960000256697336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,3,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,7,0.012432000289360682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,15,0.01101333275437355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,15,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,31,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,31,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,255,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,63,0.010837333897749582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,63,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,511,0.013429333766301474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,127,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,127,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,255,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,511,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,1023,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,1023,0.014069333672523499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,2047,0.01368533323208491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,8191,0.018874666343132656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,2047,0.015344000111023584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,4095,0.015098666151364645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,4095,0.01621866722901662
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,8191,0.01740266631046931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,16383,0.022448000808556873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,3,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,16383,0.02418133368094762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,float16,32767,0.03161599983771642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,7,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,1,128,1,float16,fp8,32767,0.03470933437347412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1,0.011061333119869232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,31,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,1,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,3,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,7,0.011242666592200598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,15,0.010928000013033548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,15,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,31,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,255,0.013034666577974955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,63,0.010938666760921478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,63,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,127,0.011114666859308878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,127,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,255,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,2047,0.01563199982047081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,511,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,511,0.014005333185195923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,1023,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,1023,0.014384000251690546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,2047,0.014170666535695394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,4095,0.015856000284353893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,32767,0.032655999064445496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,32767,0.0349386657277743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,4095,0.01706133286158244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,8191,0.018272000054518383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,8191,0.019386666516462963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,float16,16383,0.023333333432674408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,2,128,1,float16,fp8,16383,0.02479466547568639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1,0.011168000598748526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,15,0.011301333705584208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,1,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,3,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,3,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,7,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,7,0.012229333321253458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,15,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,31,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,31,0.01249066616098086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,63,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,511,0.012453333785136541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,63,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,127,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,127,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,255,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,255,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,511,0.014368000129858652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,1023,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,1023,0.015072000523408255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,8191,0.01979200045267741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,2047,0.015146666516860327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,2047,0.016735999534527462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,4095,0.01607999950647354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,4095,0.017290666699409485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,8191,0.018277333428462345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,16383,0.024480000138282776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,16383,0.02532266577084859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,float16,32767,0.03861333429813385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,4,128,1,float16,fp8,32767,0.037248000502586365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1,0.011653333902359009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,1,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,3,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,3,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,7,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,7,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,15,0.011535999675591787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,15,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,31,0.011498666057984034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,31,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,63,0.011402666568756104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,63,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,511,0.014858666807413101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,1023,0.013525333255529404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,127,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,127,0.013258667041858038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,255,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,255,0.014405333747466406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,511,0.01340266689658165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,4095,0.017935999979575474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,1023,0.014837333311637243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,2047,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,16383,0.03742400060097376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,2047,0.016666666915019352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,4095,0.01666133354107539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,8191,0.019765333582957584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,8191,0.020448000480731327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,16383,0.026901334524154663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,float16,32767,0.05905599892139435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,3,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,24,8,128,1,float16,fp8,32767,0.04628799855709076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1,0.01091733326514562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,1,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,3,0.010928000013033548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,31,0.01080000028014183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,7,0.010944000134865442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,7,0.012538666526476542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,15,0.011242666592200598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,15,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,31,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,255,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,63,0.011215999722480774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,63,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,127,0.010949333508809408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,1023,0.01482133318980535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,2047,0.014922666052977243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,127,0.013167999684810638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,255,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,511,0.012661332885424295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,511,0.013461332768201828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,1023,0.013898666948080063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,2047,0.015509333461523056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,4095,0.017450666675964992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,4095,0.018522666146357853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,8191,0.020303999384244282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,8191,0.02309333284695943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,16383,0.02961066613594691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,16383,0.032831999162832894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,3,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,float16,32767,0.047637333472569786
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,1,128,1,float16,fp8,32767,0.05169600248336792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,15,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,1,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,3,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,7,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,7,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,15,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,31,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,255,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,255,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,31,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,63,0.011514666179815928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,63,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,127,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,2047,0.01602666700879733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,127,0.01339200014869372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,511,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,511,0.014655999839305878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,8191,0.02160000056028366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,1023,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,16383,0.031231999397277832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,1023,0.01569066693385442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,2047,0.01681600014368693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,4095,0.01811733345190684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,4095,0.019274666905403137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,8191,0.024058667321999867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,16383,0.03433600068092346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,float16,32767,0.05049600203831991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,7,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,2,128,1,float16,fp8,32767,0.05369600156943003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,1,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,3,0.011488000551859537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,3,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,7,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,15,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,15,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,31,0.011477333803971609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,31,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,63,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,511,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,63,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,127,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,127,0.01357866699496905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,2047,0.016127999871969223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,255,0.012469333906968435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,255,0.014106666048367819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,511,0.014463999619086584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,1023,0.014864000181357065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,1023,0.015578666081031164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,2047,0.017045332739750545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,16383,0.03575466573238373
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,4095,0.018165333817402523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,32767,0.07193066676457723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,4095,0.019189332922299702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,8191,0.021712000171343487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,8191,0.02404266595840454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,float16,16383,0.037952000896135964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,4,128,1,float16,fp8,32767,0.05890133480230967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1,0.011541333049535751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,1,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,3,0.011503999431928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,15,0.012485332787036896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,31,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,3,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,7,0.011503999431928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,7,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,15,0.01181866725285848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,127,0.014090667168299357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,31,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,63,0.011786667009194693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,511,0.013898666948080063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,63,0.012373333175977072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,127,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,255,0.013359999905029932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,255,0.014384000251690546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,511,0.014912000546852747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,4095,0.019786667078733444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,1023,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,1023,0.016415999581416447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,2047,0.016336000214020412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,2047,0.01691199963291486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,4095,0.019253333409627277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,8191,0.03737599899371465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,8191,0.02489600082238515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,16383,0.0572213331858317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,16383,0.04558399816354116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,float16,32767,0.10042666395505269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,24,8,128,1,float16,fp8,32767,0.07335466643174489
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1,0.012847999731699625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,1,0.013525333255529404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,15,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,3,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,3,0.013125333935022354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,7,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,7,0.013232000172138214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,15,0.013397333522637686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,31,0.012597333639860153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,31,0.013557333499193192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,63,0.01314666618903478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,63,0.013424000392357508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,127,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,511,0.017968000223239262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,127,0.014682666709025701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,255,0.01488000030318896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,255,0.015509333461523056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,511,0.01626666635274887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,1023,0.021546666820844013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,1023,0.022815999885400135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,2047,0.03921066721280416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,1,0.014010666559139887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,2047,0.03452266752719879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,float16,4095,0.05876266459623972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,1,128,1,float16,fp8,4095,0.05793599784374237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1,0.0129120002190272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,3,0.013114667187134424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,3,0.014218666901191076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,7,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,7,0.014101333916187286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,15,0.012954667210578918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,15,0.014208000153303146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,31,0.013189333180586496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,31,0.0141546664138635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,63,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,63,0.014303999642531076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,127,0.01350933313369751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,511,0.018725333114465077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,1023,0.03569599986076355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,127,0.015226667126019796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,255,0.014938666174809137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,255,0.015989333391189575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,511,0.01785600061217944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,1023,0.024570666253566742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,2047,0.05750933289527893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,2047,0.04136000076929728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,float16,4095,0.09918399651845296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,3,0.018160000443458557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,2,128,1,float16,fp8,4095,0.06437866886456807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1,0.016810666769742966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,7,0.018085333208243053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,1,0.018085333208243053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,3,0.0170666662355264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,7,0.01700266698996226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,63,0.017130666722853977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,15,0.01722666621208191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,63,0.018170667191346485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,15,0.018085333208243053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,31,0.017130666722853977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,31,0.01821333294113477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,127,0.017360000560681026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,127,0.02070933332045873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,255,0.02202133337656657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,255,0.02250133454799652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,511,0.03801066676775614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,511,0.028138667345046997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,1023,0.05972800155480703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,1023,0.04437866806983948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,2047,0.10027199983596802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,2047,0.06837333242098491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,float16,4095,0.18437333901723227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1,0.024288001159826916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,4,128,1,float16,fp8,4095,0.11569066842397054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,1,0.026101333399613697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,7,0.025248001019159954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,3,0.02364266663789749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,3,0.025418666501839954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,31,0.024031999210516613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,7,0.02404266595840454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,15,0.0242399995525678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,15,0.025450666745503742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,31,0.02532800038655599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,63,0.024080000817775726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,63,0.025797332326571148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,127,0.025621332228183746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,127,0.029818666477998097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,255,0.04247466723124186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,255,0.03427733232577642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,511,0.06206933160622915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,511,0.050442665815353394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,1023,0.10251733660697937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,1023,0.0734506646792094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,4095,0.2109066645304362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,2047,0.18489599227905273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,3,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,7,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,fp8,2047,0.12018666664759318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,24,8,128,1,float16,float16,4095,0.3534239927927653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,31,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,1,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,3,0.010816000401973724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,7,0.011071999867757162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,127,0.010847999403874079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,15,0.010757333288590113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,15,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,31,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,63,0.011754666765530905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,63,0.010928000013033548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,127,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,1023,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,255,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,255,0.013280000537633896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,511,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,511,0.014607999473810196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,1023,0.014544000228246054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,8191,0.0322826678554217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,2047,0.015498666713635126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,2047,0.01785600061217944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,4095,0.020154666155576706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,4095,0.022831998765468597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,8191,0.028351999819278717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,16383,0.047456001242001854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,3,0.011173332730929056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,16383,0.051226665576299034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,7,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,float16,32767,0.0828959991534551
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,1,128,1,float16,fp8,32767,0.09125333031018575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1,0.010885333021481832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,1,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,3,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,63,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,7,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,15,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,15,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,31,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,255,0.013733333597580591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,511,0.013834666460752487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,31,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,63,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,127,0.011018666128317514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,127,0.013264000415802002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,255,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,511,0.015317333241303762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,1023,0.015413332730531693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,8191,0.030266667405764263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,1023,0.016154666741689045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,2047,0.016613333175579708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,16383,0.05579733351866404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,32767,0.08682666222254436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,32767,0.09991466999053955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,2047,0.018698666244745255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,4095,0.020687999824682873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,4095,0.023018665611743927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,fp8,8191,0.033071999748547874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,2,128,1,float16,float16,16383,0.05102399984995524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1,0.011717333147923151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,1,0.012421333541472753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,3,0.01163200040658315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,31,0.01137599969903628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,3,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,7,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,7,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,15,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,127,0.014069333672523499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,15,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,31,0.012506666282812754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,63,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,63,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,127,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,255,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,255,0.01440000037352244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,511,0.014581333845853806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,511,0.01618133361140887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,1023,0.016202667107184727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,1023,0.015957333147525787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,2047,0.016943999876578648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,2047,0.018677332748969395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,4095,0.024501333634058636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,4095,0.02330133318901062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,8191,0.03753600021203359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,8191,0.03618133316437403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,16383,0.06081066528956095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,16383,0.05842133363087972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,float16,32767,0.1097813347975413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,4,128,1,float16,fp8,32767,0.10087999701499939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1,0.011936000237862269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,1,0.012789333860079447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,3,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,3,0.012794667234023413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,7,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,7,0.012842666357755661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,15,0.011855999628702799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,15,0.012645332763592402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,127,0.0141546664138635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,31,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,31,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,511,0.01481066644191742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,63,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,63,0.012730666746695837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,127,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,255,0.013264000415802002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,255,0.014282666146755219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,511,0.015743999431530636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,4095,0.02478400121132533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,8191,0.05718400080998739
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,1023,0.016165333489576977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,1023,0.016554666062196095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,2047,0.017738666385412216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,2047,0.01882133384545644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,4095,0.03634133438269297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,8191,0.04154666761557261
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,16383,0.0997173289457957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,16383,0.07257600128650665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,float16,32767,0.1841119925181071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,24,8,128,1,float16,fp8,32767,0.11045333743095398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1,0.014202666779359182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,7,0.014202666779359182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,1,0.015066667149464289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,3,0.0141546664138635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,15,0.01421333352724711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,3,0.015471999843915304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,31,0.01551466683546702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,7,0.015322666615247726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,15,0.015317333241303762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,31,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,63,0.014117332796255747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,63,0.01540800059835116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,127,0.01470400020480156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,127,0.016602666427691776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,255,0.017792000124851864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,255,0.01735466718673706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,511,0.018746666610240936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,511,0.02021866664290428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,1023,0.037445334096749626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,1023,0.026890667776266735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,1,0.0189280000825723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,float16,2047,0.058117335041364036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,1,128,1,float16,fp8,2047,0.042992000778516136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1,0.01775466650724411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,3,0.01766933376590411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,3,0.019199999670187633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,7,0.01749333366751671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,7,0.018885333091020584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,15,0.017642666896184284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,15,0.018677332748969395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,31,0.017535999417304993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,31,0.019082666685183842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,63,0.01785600061217944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,63,0.01903466631968816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,127,0.0179626668492953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,127,0.02197866638501485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,255,0.02189333240191142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,255,0.023071999351183575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,1023,0.061119998494784035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,511,0.03935466706752777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,511,0.02959999938805898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,1023,0.046949331959088646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,1,0.025909334421157837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,fp8,2047,0.07082666456699371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1,0.024245334168275196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,2,128,1,float16,float16,2047,0.10261866450309753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,3,0.024570666253566742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,15,0.024357333779335022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,3,0.02607999990383784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,31,0.024432001014550526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,7,0.02481599897146225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,7,0.02643200010061264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,15,0.02622399975856145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,31,0.026005332668622334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,63,0.024666666984558105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,63,0.026159999271233875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,127,0.026954665780067444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,127,0.03102933367093404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,255,0.04377066592375437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,511,0.06401066482067108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,255,0.03726933399836222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,511,0.05243733525276184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,1023,0.104010671377182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,float16,2047,0.18702399730682373
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,1023,0.07472000022729237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,4,128,1,float16,fp8,2047,0.12350933750470479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1,0.037274666130542755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,1,0.040005333721637726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,7,0.03783999880154928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,3,0.037402667105197906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,15,0.03794133414824804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,15,0.039834665755430855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,31,0.03764266769091288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,63,0.03929600119590759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,31,0.03982399900754293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,3,0.039893334110577904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,63,0.040149333576361336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,7,0.04031999905904134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,127,0.04731200138727824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,127,0.048997332652409874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,1023,0.130431999762853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,1023,0.1902880072593689
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,255,0.06905599931875865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,255,0.06420266628265381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,2047,0.353765328725179
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,2047,0.22394132614135742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,float16,511,0.1086133321126302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,1,0.020597333709398907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,24,8,128,1,float16,fp8,511,0.08563733100891113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,1,0.021829334398110706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,3,0.020634666085243225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,3,0.02216000109910965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,7,0.020773333807786305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,7,0.02197333425283432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,15,0.022074667116006214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,31,0.02197866638501485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,15,0.020597333709398907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,63,0.02081599955757459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,63,0.022543999056021374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,31,0.020453333854675293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,127,0.02160533269246419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,127,0.024522667129834492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,255,0.02606933315594991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,255,0.02661866694688797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,float16,511,0.04419733087221781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,1,0.027221334477265675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,1,128,1,float16,fp8,511,0.03357866654793421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,1,0.02880000074704488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,3,0.027285332481066387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,3,0.028575999041398365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,7,0.02682666728893916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,7,0.02863999952872594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,15,0.026757332185904186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,15,0.028927999238173168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,31,0.02693866689999898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,31,0.02882133424282074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,63,0.027461332579453785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,63,0.028858666618665058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,127,0.0332640012105306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,127,0.029802667597929638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,255,0.047269334395726524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,255,0.038560000558694206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,float16,511,0.06821866830190022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,2,128,1,float16,fp8,511,0.055973331133524575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,1,0.040661332507928215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,1,0.042223999897638954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,3,0.039936001102129616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,3,0.042853335539499916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,7,0.039877332746982574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,7,0.04298666616280874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,15,0.03996799886226654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,15,0.04249600072701772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,31,0.04043200115362803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,31,0.042624001701672874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,63,0.043162668744723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,63,0.04307200014591217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,127,0.04878933231035868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,127,0.054602667689323425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,255,0.06972266733646393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,255,0.06644266843795776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,1,0.06582933167616527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,1,0.07027733325958252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,3,0.06525333225727081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,3,0.07049599786599477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,7,0.06533866624037425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,float16,511,0.11371733744939168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,7,0.06938666601975758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,4,128,1,float16,fp8,511,0.09064533313115437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,15,0.066021333138148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,15,0.0701333334048589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,31,0.06817066669464111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,31,0.07043733199437459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,63,0.07187733550866444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,127,0.07724266747633617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,127,0.0938933293024699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,63,0.0735093355178833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,255,0.1172106663386027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,1,0.0371573343873024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,1,0.03870933254559835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,255,0.10994133353233337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,3,0.03769599894682566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,float16,511,0.20106667280197144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,24,8,128,1,float16,fp8,511,0.1591146687666575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,3,0.03885866701602936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,7,0.037258667250474296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,7,0.038704000413417816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,15,0.038805333276589714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,15,0.037274666130542755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,31,0.03782399992148081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,63,0.039120001097520195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,127,0.041152000427246094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,127,0.043050666650136314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,31,0.03913066784540812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,255,0.05665599803129832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,1,0.049360002080599465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,fp8,255,0.05141866703828176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,3,0.05116266508897146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,1,128,1,float16,float16,63,0.03756800045569738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,1,0.05184000233809153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,3,0.048672000567118325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,7,0.049365331729253135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,7,0.051167999704678856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,15,0.0498933345079422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,15,0.05089599887530009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,31,0.04979733129342397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,31,0.051370665431022644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,63,0.05406400064627329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,63,0.051557332277297974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,127,0.061424002051353455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,127,0.06384000182151794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,1,0.07542400062084198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,fp8,255,0.07727999985218048
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,1,0.0803306649128596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,3,0.07565333445866902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,2,128,1,float16,float16,255,0.08258133133252461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,3,0.07996800045172374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,7,0.07606400052706401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,7,0.07977066437403361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,15,0.07578133543332417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,31,0.08064533273379008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,15,0.0795306662718455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,31,0.0811466674009959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,63,0.08468799789746602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,63,0.08413867155710857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,127,0.09047999978065491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,127,0.1029813289642334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,1,0.12590400377909342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,1,0.13451199730237326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,float16,255,0.1269706686337789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,4,128,1,float16,fp8,255,0.12102400263150533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,3,0.12614400188128153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,3,0.1347040037314097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,7,0.12819733222325644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,15,0.13152533769607544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,7,0.13226133584976196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,15,0.1349493364493052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,63,0.13593600193659464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,63,0.142085333665212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,31,0.13492266337076822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,31,0.13912000258763632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1,0.011034666250149408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,3,0.011061333119869232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,7,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,7,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,1,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,3,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,15,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,15,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,127,0.14655466874440512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,31,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,127,0.17193599541982016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,31,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,63,0.010960000256697336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,63,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,127,0.013546666751305262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,127,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,fp8,255,0.20436267058054605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,255,0.014085333794355392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,24,8,128,1,float16,float16,255,0.21817066272099814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,511,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,255,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,511,0.014975999792416891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,1023,0.015930666277805965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,1023,0.01782400036851565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,2047,0.022522665560245514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,16383,0.08373333017031352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,4095,0.03193599979082743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,8191,0.046682665745417275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,8191,0.051407997806866966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,2047,0.01993600030740102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,16383,0.09259733557701111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,4095,0.02829333394765854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,float16,32767,0.1520639955997467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,7,0.011642667154471079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,1,128,1,float16,fp8,32767,0.18598934014638266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,15,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,1,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,3,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,3,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,7,0.011850666254758835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,15,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,31,0.011589333415031433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,31,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,255,0.012997332960367203
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,63,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,63,0.011936000237862269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,127,0.011242666592200598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,127,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,255,0.015184000134468079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,511,0.014602666099866232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,511,0.016106666376193363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,1023,0.016645333419243496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,1023,0.01826133330663045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,2047,0.020448000480731327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,2047,0.0230880007147789
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,4095,0.029818666477998097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,4095,0.03349333256483078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,32767,0.15249600013097128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,8191,0.05117333432038625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,8191,0.05397333204746246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,float16,16383,0.08522666494051616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,3,0.012768000364303589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,16383,0.10068266590436299
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,7,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,2,128,1,float16,fp8,32767,0.18451732397079468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,1,0.013050666699806849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,3,0.011936000237862269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,7,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,15,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,15,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,31,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,31,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,63,0.011717333147923151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,63,0.013077333569526672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,127,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,127,0.01451733335852623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,255,0.013637332866589228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,255,0.01571200042963028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,511,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,511,0.01617066686352094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,1023,0.016517333686351776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,1023,0.01907733331123988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,2047,0.021168000996112823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,2047,0.023141334454218548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,4095,0.03712533414363861
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,4095,0.03414933383464813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,8191,0.061648001273473106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,8191,0.05798399945100149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,16383,0.11062399546305339
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,16383,0.10103999574979146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,float16,32767,0.19606933991114298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,4,128,1,float16,fp8,32767,0.18185067176818848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,7,0.013237333546082178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,1,0.012800000607967377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,3,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,3,0.012901333471139273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,7,0.012730666746695837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,15,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,127,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,15,0.013914667069911957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,31,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,31,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,63,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,511,0.015226667126019796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,63,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,127,0.014069333672523499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,255,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,255,0.015840000162522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,511,0.01743999992807706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,4095,0.04155199974775314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,1023,0.01749333366751671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,1023,0.018933333456516266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,2047,0.0365280012289683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,2047,0.026202666262785595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,4095,0.057429333527882896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,8191,0.09961066643397014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,8191,0.06449600060780843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,32767,0.2283626596132914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,16383,0.18404799699783325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,fp8,16383,0.1264746685822805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,24,8,128,1,float16,float16,32767,0.35660266876220703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,1,0.03807999938726425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,1,0.03793066740036011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,3,0.03789866715669632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,7,0.037903999288876854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,3,0.03852800031503042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,7,0.038149334490299225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,15,0.037791999677817024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,15,0.03852800031503042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,31,0.03833066672086716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,31,0.03834133346875509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,63,0.04398400088151296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,float16,127,0.04855466882387797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,1,0.06387733419736226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,63,0.03919466584920883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,1,128,1,float16,fp8,127,0.05194666484991709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,1,0.06295999884605408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,3,0.06342933575312297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,3,0.06187733511130015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,7,0.06414400041103363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,7,0.06204266846179962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,15,0.06477866570154826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,31,0.07074133555094402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,15,0.06300800045331319
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,31,0.06334400177001953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,63,0.07178133229414622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,63,0.0713866651058197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,float16,127,0.07865066826343536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,1,0.11454400420188904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,2,128,1,float16,fp8,127,0.08736000458399455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,1,0.11211199561754863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,3,0.11559466520945232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,3,0.11347732941309611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,7,0.11545600493748982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,7,0.11218133568763733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,15,0.12657066186269125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,15,0.1129866639773051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,31,0.12652800480524698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,31,0.12174399693806966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,63,0.12922666470209757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,63,0.1265066663424174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,float16,127,0.13634666800498962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,1,0.2121386726697286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,1,0.20535999536514282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,4,128,1,float16,fp8,127,0.1571946640809377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,3,0.21118932962417603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,3,0.20495466391245523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,7,0.22853867212931314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,15,0.2243573268254598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,7,0.2087786595026652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,31,0.22951465845108032
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,15,0.22688533862431845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,31,0.22057066361109415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,63,0.2309066653251648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,63,0.22294400135676065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,float16,127,0.24833599726359049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,24,8,128,1,float16,fp8,127,0.2928853432337443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,1,0.06596800188223521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,1,0.06528000036875407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,3,0.06525333225727081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,3,0.06547733147939046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,7,0.06736533343791962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,7,0.06588266789913177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,15,0.0662720004717509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,15,0.07275733351707458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,31,0.07222933570543925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,31,0.07479466497898102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,float16,63,0.07461866736412048
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,1,128,1,float16,fp8,63,0.07712000111738841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,1,0.11588799953460693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,1,0.11287466684977214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,3,0.1209279994169871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,3,0.11315733194351196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,7,0.12731200456619263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,15,0.12805333733558655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,15,0.11758933464686076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,31,0.12882133324941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,31,0.12599466244379678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,float16,63,0.1309440036614736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,7,0.11406933267911275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,1,0.22524267435073853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,2,128,1,float16,fp8,63,0.12812800208727518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,1,0.212826669216156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,3,0.2342080076535543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,3,0.2130826711654663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,7,0.2387626568476359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,7,0.22549867630004883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,15,0.24074133237202963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,15,0.23188799619674683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,31,0.23121066888173422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,1,0.43993067741394043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,31,0.24116265773773193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,float16,63,0.2412266731262207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,4,128,1,float16,fp8,63,0.23275200525919595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,1,0.4058506488800049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,3,0.44597868124643963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,3,0.41942934195200604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,7,0.4390559991200765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,7,0.428058664004008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,31,0.44236799081166583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1,0.011663999408483505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,15,0.4464319944381714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,1,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,15,0.42353065808614093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,3,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,7,0.011637333780527115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,15,0.011653333902359009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,15,0.012522666404644648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,7,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,3,0.011338666081428528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,31,0.011472000430027643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,31,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,63,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,63,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,31,0.4293493429819743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,127,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,127,0.013471999516089758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,255,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,255,0.014261333892742792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,511,0.01481066644191742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,511,0.016858667135238647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,1023,0.01922133316596349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,1023,0.021727999051411945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,2047,0.03120533376932144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,2047,0.02771199991305669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,4095,0.04615999758243561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,4095,0.05061866839726766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,8191,0.08288000027338664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,8191,0.09092799822489421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,float16,16383,0.1500640014807383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,1,128,1,float16,fp8,16383,0.18254399299621582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,1,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,3,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,3,0.012906666845083237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,7,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,7,0.012789333860079447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,15,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,63,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,15,0.013007999708255133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,31,0.012229333321253458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,31,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,float16,63,0.4535573323567708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,63,0.012826666235923767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,127,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,127,0.014058666924635569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,255,0.01358933374285698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,255,0.014965333044528961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,511,0.015413332730531693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,511,0.017445333302021027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,1023,0.021930667261282604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,2047,0.028533334533373516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,2047,0.03196266790231069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,4095,0.04909333089987437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,1023,0.020266667008399963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,4095,0.05198400219281515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,8191,0.08455999692281087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,24,8,128,1,float16,fp8,63,0.43044265111287433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,1,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,8191,0.09853333234786987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,float16,16383,0.15524799625078836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,2,128,1,float16,fp8,16383,0.18265066544214884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,3,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,3,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,7,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,15,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,63,0.013125333935022354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,15,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,31,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,7,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,31,0.013349333157142004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,63,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,127,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,127,0.014159999787807465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,255,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,255,0.015061333775520325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,511,0.015813333292802174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,511,0.017456000049908955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,1023,0.021151999632517498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,1023,0.02239999920129776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,2047,0.03586666782697042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,2047,0.033904001116752625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,4095,0.05958400170008341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,4095,0.05686933298905691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,8191,0.10122666756312053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,8191,0.09903466701507568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,float16,16383,0.1871359944343567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,4,128,1,float16,fp8,16383,0.1805866758028666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,1,0.013104000439246496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,3,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,3,0.013066666821638743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,7,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,63,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,7,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,15,0.012618667135636011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,127,0.014831999937693277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,255,0.014485333114862442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,15,0.01310933381319046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,31,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,31,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,63,0.013365333278973898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,1023,0.023743999501069386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,127,0.012991999586423239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,255,0.015546667079130808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,511,0.016704000532627106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,511,0.01803733284274737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,1023,0.03555200000603994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,2047,0.056133334835370384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,2047,0.03967999915281931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,4095,0.09841066598892212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,4095,0.06277333199977875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,8191,0.18276800711949667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,8191,0.10918933153152466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,float16,16383,0.35184534390767414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,24,8,128,1,float16,fp8,16383,0.19934932390848795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1,0.011749333391586939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,1,0.012661332885424295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,3,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,3,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,7,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,7,0.012506666282812754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,15,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,15,0.012549333274364471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,31,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,31,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,63,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,63,0.012602667013804117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,127,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,127,0.014501333236694336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,255,0.013536000003417334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,255,0.014688000082969666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,511,0.015520000209410986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,511,0.01724799970785777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,1023,0.01969066634774208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,4095,0.05337599913279215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,8191,0.0832426647345225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,1023,0.021914665897687275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,2047,0.02974933385848999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,16383,0.1557866632938385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,2047,0.03161599983771642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,float16,4095,0.04915733138720194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,8191,0.0995199978351593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,1,128,1,float16,fp8,16383,0.18549867471059164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,1,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,15,0.013013333082199097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,3,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,3,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,31,0.013157332936922709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,7,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,7,0.013141332815090815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,15,0.011786667009194693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,31,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,63,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,63,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,127,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,127,0.014122666170199713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,255,0.014256000518798828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,255,0.015168000012636185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,511,0.015765332927306492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,511,0.017397332936525345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,1023,0.020432000358899433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,1023,0.022245332598686218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,8191,0.10618666807810466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,8191,0.0951039989789327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,2047,0.03618133316437403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,2047,0.03297599901755651
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,4095,0.05884266893068949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,4095,0.05451733370621999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,3,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,7,0.012522666404644648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,float16,16383,0.18663465976715088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,2,128,1,float16,fp8,16383,0.17132800817489624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,1,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,3,0.013194666554530462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,7,0.013295999417702356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,63,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,15,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,15,0.013621332744757334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,31,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,31,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,63,0.013445333888133367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,127,0.012896000097195307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,127,0.014762666076421738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,255,0.014373333503802618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,255,0.015669333438078564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,511,0.018197332819302876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,511,0.018021332720915478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,4095,0.09877333045005798
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,1023,0.0347626656293869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,1023,0.023562667270501454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,2047,0.056602666775385536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,2047,0.040661332507928215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,4095,0.06381333371003468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,8191,0.18305067221323648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,8191,0.10980266332626343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,3,0.016575999557971954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,float16,16383,0.3522506554921468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,1,0.01746133342385292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1,0.016586666305859882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,4,128,1,float16,fp8,16383,0.20219733317693075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,3,0.017866666118303936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,7,0.016607999801635742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,7,0.01757866640885671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,63,0.016602666427691776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,15,0.016480000068744022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,15,0.017338667064905167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,31,0.016613333175579708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,31,0.01758933315674464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,63,0.017903999735911686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,127,0.017077332983414333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,127,0.02033599962790807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,255,0.020549333343903225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,255,0.021594665944576263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,511,0.03757333258787791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,511,0.027776000400384266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,2047,0.06696000198523204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,1023,0.058506667613983154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,4095,0.1849600076675415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,1023,0.04366933306058248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,2047,0.09969600041707356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,4095,0.11264000336329143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,8191,0.20379199584325156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1,0.011050666371981302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,1,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,8191,0.3501173257827759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,float16,16383,0.6882346471150717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,16,8,128,1,float16,fp8,16383,0.3839999834696452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,3,0.01098666712641716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,3,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,7,0.011173332730929056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,7,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,15,0.010949333508809408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,15,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,127,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,255,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,31,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,31,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,63,0.010869332899649939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,63,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,127,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,255,0.013264000415802002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,511,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,511,0.013541333377361298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,1023,0.012645332763592402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,1023,0.014346666634082794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,8191,0.018911999960740406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,2047,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,2047,0.014997333288192749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,4095,0.01532799998919169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,4095,0.016330666840076447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,8191,0.017349333812793095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,16383,0.0220320001244545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,16383,0.024661332368850708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,32767,0.031845333675543465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,32767,0.034304000437259674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,float16,65535,0.05093866586685181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,1,128,1,float16,fp8,65535,0.05552533268928528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,1,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,3,0.01090666651725769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,3,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,7,0.011007999380429586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,7,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,15,0.010890666395425797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,15,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,31,0.011178666104873022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,255,0.011920000116030375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,31,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,63,0.01099733387430509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,63,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,127,0.011066666493813196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,127,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,255,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,511,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,511,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,1023,0.012885333349307379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,8191,0.018266666680574417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,1023,0.013850666582584381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,2047,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,2047,0.015504000087579092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,4095,0.015930666277805965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,4095,0.01721599946419398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,8191,0.019050666441520054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,16383,0.022687998910744984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,16383,0.02454400062561035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,32767,0.03219733387231827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,32767,0.0351200004418691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,3,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,float16,65535,0.05375466744105021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,2,128,1,float16,fp8,65535,0.056885331869125366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,1,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,3,0.011061333119869232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,7,0.011066666493813196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,7,0.012549333274364471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,63,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,15,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,15,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,31,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,31,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,511,0.012479999413092932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,63,0.011402666568756104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,127,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,127,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,255,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,255,0.0136266661187013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,511,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,1023,0.014240000396966934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,1023,0.015157333264748255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,2047,0.014917333920796713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,2047,0.016538667182127636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,4095,0.016666666915019352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,4095,0.01693333312869072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,32767,0.045653333266576133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,8191,0.01811733345190684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,8191,0.020010666300853092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,16383,0.02422933280467987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,16383,0.025248001019159954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,1,0.012256000190973282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,32767,0.03756800045569738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,float16,65535,0.06078400214513143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,4,128,1,float16,fp8,65535,0.063360000650088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,3,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,3,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,7,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,7,0.012304000556468964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,15,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,15,0.012469333906968435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,31,0.011557333171367645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,255,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,31,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,63,0.011365332951148352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,63,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,127,0.011253333340088526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,127,0.013269333789745966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,2047,0.015061333775520325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,255,0.01413333291808764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,511,0.013503999759753546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,4095,0.017573333034912746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,511,0.01488000030318896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,1023,0.013525333255529404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,16383,0.036837334434191384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,1023,0.014901333798964819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,32767,0.059445331494013466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,2047,0.016613333175579708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,4095,0.016890666137139004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,8191,0.020853333175182343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,8191,0.020341333001852036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,16383,0.026677332818508148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,32767,0.04218666752179464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,float16,65535,0.10088533163070679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,16,8,128,1,float16,fp8,65535,0.07461333274841309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1,0.011066666493813196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,1,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,3,0.011061333119869232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,3,0.01258133351802826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,7,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,63,0.010911999891201654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,7,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,15,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,127,0.013269333789745966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,15,0.012597333639860153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,31,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,31,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,511,0.01351999988158544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,63,0.012874666601419449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,127,0.011007999380429586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,255,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,255,0.013167999684810638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,511,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,1023,0.01351999988158544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,1023,0.01469333345691363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,2047,0.014736000448465347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,16383,0.02884799987077713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,2047,0.01602666700879733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,4095,0.017071999609470367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,4095,0.01854933301607768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,8191,0.020389333367347717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,8191,0.023130667706330616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,16383,0.0332640012105306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,32767,0.04641066491603851
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,32767,0.05189866820971171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,float16,65535,0.08391466736793518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,1,128,1,float16,fp8,65535,0.09215999643007915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,1,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,3,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,3,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,7,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,7,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,15,0.011637333780527115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,127,0.01129066695769628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,15,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,31,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,31,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,63,0.011402666568756104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,511,0.013914667069911957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,63,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,127,0.012938667088747025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,2047,0.016095999628305435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,255,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,255,0.013242666920026144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,511,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,4095,0.0191040001809597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,1023,0.013679999858140945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,1023,0.015664000064134598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,16383,0.03053866575161616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,2047,0.016336000214020412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,4095,0.017429333180189133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,8191,0.02123733361562093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,8191,0.023157333334287006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,65535,0.09422933061917622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,16383,0.033589333295822144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,32767,0.04844266672929128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,3,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,fp8,32767,0.05471999943256378
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,2,128,1,float16,float16,65535,0.08418666323026021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,7,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1,0.01128000020980835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,1,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,3,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,63,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,63,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,7,0.011381333072980246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,15,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,255,0.012453333785136541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,15,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,31,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,511,0.0145066666106383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,31,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,127,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,127,0.013264000415802002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,255,0.014015999933083853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,511,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,1023,0.015173333386580149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,8191,0.023071999351183575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,1023,0.01616000011563301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,2047,0.016373333831628162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,2047,0.016597333053747814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,4095,0.0183146670460701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,4095,0.019727999965349834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,8191,0.023552000522613525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,16383,0.0429066667954127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1,0.011557333171367645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,1,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,16383,0.034346667428811394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,32767,0.059994667768478394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,32767,0.0584853341182073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,float16,65535,0.1090666651725769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,4,128,1,float16,fp8,65535,0.10252267122268677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,3,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,31,0.011546666423479715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,31,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,3,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,7,0.011365332951148352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,7,0.012245333443085352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,15,0.011706666400035223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,15,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,63,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,63,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,511,0.014853333433469137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,127,0.011391999820868174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,127,0.013888000200192133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,255,0.013269333789745966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,255,0.014538666854302088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,4095,0.018559999763965607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,511,0.013845333208640417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,1023,0.015658666690190632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,8191,0.026949333647886913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,1023,0.01623999948302905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,2047,0.016735999534527462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,2047,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,4095,0.019589333484570186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,8191,0.03669333209594091
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,16383,0.058117335041364036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,16383,0.04587733248869578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,32767,0.10032533605893452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,32767,0.06392533580462138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,3,0.012794667234023413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,float16,65535,0.1841920018196106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,16,8,128,1,float16,fp8,65535,0.11087999741236369
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,15,0.012736000120639801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,1,0.013104000439246496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,3,0.01333333303531011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,7,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,7,0.013295999417702356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,15,0.013461332768201828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,31,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,31,0.013258667041858038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,63,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,63,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,127,0.013306666165590286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,511,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,127,0.014629332969586054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,255,0.014581333845853806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,255,0.01523200049996376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,2047,0.03640000025431315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,511,0.01623999948302905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,1023,0.02126399924357732
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,1023,0.022597332795461018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,2047,0.0328053335348765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,4095,0.061568001906077065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,4095,0.05769599974155426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,float16,8191,0.11012267072995503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,3,0.012960000584522883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,1,128,1,float16,fp8,8191,0.10194666186968486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,3,0.013914667069911957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1,0.013232000172138214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,1,0.01370666672786077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,7,0.013050666699806849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,7,0.013738666971524557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,15,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,15,0.013781332721312841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,31,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,31,0.013754667093356451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,127,0.014709333578745524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,63,0.013386666774749756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,63,0.014058666924635569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,127,0.01403733342885971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,511,0.01865600049495697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,255,0.016309333344300587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,1023,0.02430933217207591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,255,0.015802666544914246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,2047,0.04029866556326548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,511,0.01794133335351944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,1023,0.03540800015131632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,2047,0.05671466886997223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,4095,0.09894933303197224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,4095,0.06399466594060262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,float16,8191,0.18361065785090128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,2,128,1,float16,fp8,8191,0.1107306679089864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,1,0.018218666315078735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,7,0.01820266619324684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,15,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1,0.01714666684468587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,3,0.016943999876578648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,31,0.0182239996890227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,7,0.017242666333913803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,3,0.0183146670460701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,15,0.018085333208243053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,31,0.017018667111794155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,63,0.017162666966517765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,63,0.018031999468803406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,127,0.017445333302021027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,127,0.021338666478792827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,255,0.021573332448800404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,255,0.02242133269707362
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,511,0.037503999968369804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,511,0.029674666623274486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,1023,0.05938133100668589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,2047,0.10106133421262105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,1023,0.04562133550643921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,2047,0.06854400038719177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,4095,0.18519467115402222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,4095,0.11521066228548686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,float16,8191,0.3508960008621216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,4,128,1,float16,fp8,8191,0.2070293426513672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,1,0.02515733242034912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1,0.024175999065240223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,3,0.02378133436044057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,3,0.02552533398071925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,31,0.023845332364241283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,7,0.02438933402299881
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,7,0.025279998779296875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,63,0.024112001061439514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,15,0.023749334116776783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,15,0.02555199960867564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,31,0.025306666890780132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,63,0.02565866708755493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,127,0.024485332270463307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,127,0.03030933439731598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,511,0.050474668542544045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,255,0.04171733558177948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,255,0.03436266630887985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,511,0.06053866446018219
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,1023,0.10285333792368571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,1023,0.07353066901365916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,4095,0.21024533112843832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,2047,0.1840533415476481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,2047,0.1199733316898346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,4095,0.3522293170293172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1,0.01091733326514562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,1,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,3,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,float16,8191,0.6872639656066895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,3,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,16,8,128,1,float16,fp8,8191,0.3891040086746216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,7,0.01101333275437355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,7,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,15,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,15,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,31,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,31,0.011584000041087469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,255,0.01333333303531011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,63,0.01090666651725769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,63,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,127,0.011034666250149408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,127,0.012837332983811697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,255,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,511,0.013424000392357508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,511,0.014005333185195923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,1023,0.01479999969402949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,1023,0.015216000378131866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,2047,0.015541333705186844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,2047,0.018133333573738735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,16383,0.04673066735267639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,4095,0.020021333048741024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,4095,0.022682666778564453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,8191,0.02827200045188268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,8191,0.032645332316557564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,16383,0.051311999559402466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,32767,0.08410132924715678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,32767,0.09582933783531189
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,float16,65535,0.15317866206169128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,1,128,1,float16,fp8,65535,0.18463999032974243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1,0.010928000013033548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,1,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,15,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,3,0.01097600037852923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,3,0.011802667131026586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,7,0.010821333775917688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,63,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,7,0.011530666301647821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,15,0.011045332998037338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,31,0.011055999745925268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,31,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,63,0.011221333096424738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,127,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,127,0.012730666746695837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,1023,0.015717333803574245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,255,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,255,0.013530666629473368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,511,0.01394133393963178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,511,0.014890667051076889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,8191,0.030271999537944794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,1023,0.015765332927306492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,2047,0.01637866720557213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,2047,0.01800000046690305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,4095,0.02057066683967908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,32767,0.09567999839782715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,4095,0.022672000030676525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,8191,0.03281066566705704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,16383,0.04794666667779287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,1,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,16383,0.05343999962011973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,3,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,32767,0.08451199531555176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,float16,65535,0.15170666575431824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,2,128,1,float16,fp8,65535,0.17176532745361328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,3,0.013797332843144735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,7,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,7,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,15,0.012730666746695837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,15,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,31,0.011413333316644033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,31,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,255,0.014090667168299357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,63,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,63,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,127,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,1023,0.016421332955360413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,127,0.013658666362365087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,255,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,511,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,511,0.015392000476519266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,8191,0.03669866671164831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,1023,0.01571200042963028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,2047,0.017071999609470367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,2047,0.01841066653529803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,4095,0.02462399999300639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,4095,0.023269332945346832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,8191,0.033786666889985405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,16383,0.07203733424345653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,16383,0.05865600208441416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,32767,0.12800533572832742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,3,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,32767,0.09950400392214458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,float16,65535,0.24106132984161377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,7,0.015919999529918034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,4,128,1,float16,fp8,65535,0.1845973332722982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,1,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,31,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,3,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,7,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,15,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,15,0.012661332885424295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,31,0.011621333658695221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,63,0.011477333803971609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,63,0.012709333250919977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,511,0.015578666081031164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,127,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,127,0.013690666606028875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,255,0.013679999858140945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,2047,0.01922133316596349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,255,0.01421333352724711
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,511,0.014554666976133982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,1023,0.016021333634853363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,1023,0.016261332978804905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,2047,0.017818666994571686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,4095,0.03636800001064936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,4095,0.025285333395004272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,8191,0.05754133562246958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,8191,0.04124266654253006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,16383,0.09915733337402344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,16383,0.06342400113741557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,32767,0.1835626761118571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,32767,0.10922132929166158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,float16,65535,0.35474133491516113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,16,8,128,1,float16,fp8,65535,0.23517866929372153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1,0.013898666948080063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,1,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,3,0.014149333039919535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,3,0.015205333630243937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,7,0.01394133393963178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,7,0.015295999745527903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,15,0.014138666292031607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,15,0.015018666783968607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,31,0.014096000542243322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,31,0.014933332800865173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,63,0.015306666493415833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,63,0.01422400027513504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,127,0.014815999815861383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,127,0.01646399994691213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,255,0.015829333414634068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,255,0.01709866647919019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,511,0.01974933346112569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,511,0.019861333072185516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,1023,0.036229332288106285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,1023,0.025349333882331848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,2047,0.05784533421198527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,2047,0.04273599882920583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1,0.017829333742459614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,1,0.01846933364868164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,3,0.018559999763965607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,fp8,4095,0.06637333333492279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,1,128,1,float16,float16,4095,0.10179199775060017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,3,0.017525333911180496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,7,0.01748266691962878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,7,0.01869333287080129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,15,0.01741333305835724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,15,0.01851733277241389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,31,0.01848000039656957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,63,0.01860800012946129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,31,0.017525333911180496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,127,0.017840000490347546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,63,0.017632000148296356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,127,0.021941334009170532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,255,0.022389332453409832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,255,0.022789334257443745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,511,0.03934400031963984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,511,0.029311999678611755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,1023,0.05939733485380808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,1023,0.04613333443800608
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,4095,0.1851360003153483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,float16,2047,0.10140267014503479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,4095,0.11514133214950562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,2,128,1,float16,fp8,2047,0.07029333213965099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1,0.02442666639884313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,1,0.026122666895389557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,3,0.024341332415739696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,7,0.023893333971500397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,3,0.02593066543340683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,15,0.024362665911515553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,7,0.02622399975856145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,31,0.02456533412138621
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,31,0.026021334032217663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,15,0.02628266563018163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,63,0.02463999887307485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,63,0.02586666742960612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,127,0.02593066543340683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,127,0.030368000268936157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,511,0.06397333244482677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,255,0.041637333730856575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,255,0.03427733232577642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,511,0.05260799825191498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,2047,0.18614933888117471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,1023,0.10276266932487488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,2047,0.1228053371111552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,float16,4095,0.3532533248265584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1,0.037077332536379494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,1,0.04029866556326548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,4095,0.21302932500839233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,4,128,1,float16,fp8,1023,0.07369066774845123
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,7,0.037274666130542755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,3,0.03761066744724909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,7,0.04038933416207632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,15,0.04011200120051702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,3,0.04020266731580099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,15,0.03736533224582672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,31,0.037717332442601524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,63,0.03788266579310099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,63,0.04016000032424927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,127,0.04665066798528036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,127,0.05293866495291392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,31,0.03991466760635376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,255,0.06881600121657054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,1023,0.18969066937764487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,255,0.06149866680304209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,1023,0.13030399878819784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,2047,0.22292800744374594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,511,0.08490133285522461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,2047,0.3542399803797404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,511,0.10924266775449117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,float16,4095,0.6869333585103353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1,0.019941333681344986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,16,8,128,1,float16,fp8,4095,0.40087465445200604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,1,0.02146133283774058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,3,0.019871999820073444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,3,0.02186666677395503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,7,0.02163200080394745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,15,0.020181333025296528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,7,0.020319999506076176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,15,0.02143999934196472
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,31,0.020010666300853092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,31,0.021498667697111767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,63,0.020362666497627895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,63,0.021669333179791767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,127,0.020517333100239437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,127,0.024080000817775726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,255,0.02624533325433731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,255,0.025786665578683216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,511,0.04267199834187826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,511,0.03182400017976761
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1,0.027493332823117573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,1,0.027488000690937042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,float16,1023,0.06454933186372121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,1,128,1,float16,fp8,1023,0.04980266590913137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,3,0.026170666019121807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,3,0.027488000690937042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,7,0.02590399980545044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,7,0.027466667195161183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,15,0.02640533447265625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,15,0.0276853342851003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,31,0.02685333291689555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,31,0.027877333263556164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,63,0.02651199946800868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,63,0.027552001178264618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,127,0.032085334261258446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,127,0.028586665789286297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,255,0.044453332821528115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,255,0.038149334490299225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,511,0.05470933516820272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,511,0.06704000135262807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,float16,1023,0.10593600074450175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1,0.03919466584920883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,2,128,1,float16,fp8,1023,0.07669866581757863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,1,0.04233600199222565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,3,0.03925866633653641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,3,0.04212800165017446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,7,0.039279999832312264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,15,0.04187199970086416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,31,0.042463997999827065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,63,0.041493333876132965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,7,0.0429013321797053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,63,0.04237333436806997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,31,0.03951466580231985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,127,0.04924799998601278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,15,0.040021332601706185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,127,0.05144000053405762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,255,0.06830400228500366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,255,0.06625600159168243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1,0.0659093310435613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,1,0.06857066849867503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,511,0.08949333429336548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,511,0.1128053367137909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,3,0.069733331600825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,3,0.06468266745408376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,7,0.06579733391602834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,7,0.06857066849867503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,15,0.065610667069753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,fp8,1023,0.1337493360042572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,31,0.06734933455785115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,15,0.06991466879844666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,4,128,1,float16,float16,1023,0.1928000052769979
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,31,0.06922666728496552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,63,0.0721919983625412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,63,0.07098133365313213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,127,0.07625600198904674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,255,0.11540266871452332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,127,0.09146666526794434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,511,0.1995840072631836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,255,0.10699733098347981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,1023,0.23973333835601807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,float16,1023,0.36109864711761475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,16,8,128,1,float16,fp8,511,0.15406933426856995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,1,0.03465600063403448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,1,0.03619733452796936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,3,0.03461333364248276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,7,0.03566933423280716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,3,0.0359253336985906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,7,0.03496533383925756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,15,0.03506666670242945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,31,0.034634667138258614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,31,0.03637866675853729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,63,0.03455466777086258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,15,0.03605333218971888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,63,0.03585600107908249
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,127,0.037690666814645134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,127,0.04058133314053217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,255,0.054330666859944664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,255,0.04836266736189524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,float16,511,0.0767626663049062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,1,0.045082668463389076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,1,128,1,float16,fp8,511,0.06481066842873891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,1,0.04734933376312256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,3,0.046367997924486794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,3,0.047968000173568726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,7,0.04538666705290476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,7,0.047872001926104225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,15,0.04516266783078512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,15,0.047968000173568726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,31,0.04562666515509287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,31,0.0480373352766037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,63,0.04769066472848257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,127,0.05798399945100149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,63,0.048938666780789696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,255,0.0767146646976471
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,127,0.05787200232346853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,255,0.07337599992752075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,float16,511,0.11879466970761617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,1,0.07287999987602234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,1,0.07814933359622955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,2,128,1,float16,fp8,511,0.09697066744168599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,3,0.07340266803900401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,7,0.07346133391062419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,3,0.07745066781838734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,7,0.07743466893831889
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,15,0.07776000102361043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,15,0.07291733225186665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,31,0.07567999760309856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,63,0.07946666578451793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,63,0.0801333338022232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,127,0.0860533316930135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,31,0.07813866436481476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,127,0.10076799988746643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,1,0.12356266379356384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,255,0.12392533818880717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,1,0.13015466928482056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,255,0.11801066994667053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,float16,511,0.20797866582870483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,3,0.12521066268285116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,3,0.13108799854914346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,4,128,1,float16,fp8,511,0.16832000017166138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,7,0.12359999616940816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,7,0.13126933574676514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,15,0.1270133356253306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,15,0.13031466801961264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,31,0.13598933815956116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,63,0.1337440013885498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,31,0.1325546701749166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,127,0.17058666547139487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,127,0.14206399520238241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,63,0.13963733116785684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,3,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,7,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,7,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,15,0.010992000500361124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,1,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,255,0.21453332901000977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,15,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,31,0.011034666250149408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,3,0.011055999745925268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,31,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,255,0.19941333929697672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,63,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,127,0.011034666250149408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,127,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,255,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,63,0.010874666273593903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,511,0.01379199946920077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,511,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,1023,0.015674666812022526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,fp8,511,0.29285866022109985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,16,8,128,1,float16,float16,511,0.38754133383433026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,4095,0.03190933416287104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,1023,0.017792000124851864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,255,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,2047,0.019946667055288952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,2047,0.02258133391539256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,4095,0.02868266652027766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,16383,0.08187200129032135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,16383,0.09644800424575806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,32767,0.15105066696802774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1,0.011301333705584208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,8191,0.05190933247407278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,32767,0.18292800585428873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,65535,0.2932746609052022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,float16,8191,0.04721599817276001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,7,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,1,128,1,float16,fp8,65535,0.3551413218180339
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,1,0.011706666400035223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,3,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,3,0.011674666156371435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,7,0.011429333438475927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,63,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,127,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,15,0.011594666788975397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,15,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,31,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,31,0.011733333269755045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,511,0.014655999839305878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,63,0.011477333803971609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,127,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,255,0.013242666920026144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,2047,0.02096533278624217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,255,0.014959999670584997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,511,0.015967999895413715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,1023,0.016586666305859882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,1023,0.01802666609485944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,8191,0.05332266787687937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,2047,0.022944000860055287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,4095,0.029306667546431225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,4095,0.033029332756996155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,8191,0.04868799944718679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,16383,0.08694400389989217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,16383,0.09272533655166626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,32767,0.15692266821861267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,32767,0.17062399784723917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,float16,65535,0.2940746744473775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,2,128,1,float16,fp8,65535,0.32682132720947266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,15,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,1,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,3,0.011578666667143503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,3,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,7,0.011754666765530905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,7,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,15,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,31,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,31,0.013066666821638743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,255,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,63,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,63,0.012709333250919977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,127,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,1023,0.018485333770513535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,127,0.014197333405415217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,255,0.013552000125249227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,511,0.01471466695268949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,511,0.016623999923467636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,1023,0.01693333312869072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,2047,0.021274665991465252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,2047,0.023130667706330616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,4095,0.044064000248909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,4095,0.034517332911491394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,8191,0.061253334085146584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,8191,0.057215998570124306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,16383,0.10961600144704182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,16383,0.10013332962989807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,32767,0.19614400466283163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,32767,0.18112534284591675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,3,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,float16,65535,0.46568532784779865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,4,128,1,float16,fp8,65535,0.35066131750742596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,1,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,3,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,7,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,7,0.0129120002190272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,63,0.013514666507641474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,127,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,15,0.011978667229413986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,15,0.012991999586423239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,31,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,511,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,31,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,63,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,1023,0.018757333358128864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,127,0.013973332941532135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,255,0.013914667069911957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,255,0.015872000406185787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,511,0.016970666746298473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,1023,0.0179626668492953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,2047,0.0360000009338061
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,2047,0.026005332668622334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,16383,0.1851146618525187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,4095,0.057802667220433555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,4095,0.04131199916203817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,8191,0.09986666838328044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,8191,0.06480533381303151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,16383,0.12693333625793457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,65535,0.3819253444671631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,65535,0.697989304860433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,float16,32767,0.35550399621327716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,16,8,128,1,float16,fp8,32767,0.20076799392700195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,1,0.03736533224582672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,1,0.03849600007136663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,3,0.03801066676775614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,3,0.03802666564782461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,7,0.03775999943415324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,7,0.03812266637881597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,15,0.037690666814645134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,15,0.03827200084924698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,31,0.03718400001525879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,63,0.04200000067551931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,31,0.038575999438762665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,127,0.047338664531707764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,127,0.05095999936262766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,float16,255,0.06653333206971486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,255,0.06448533137639363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,1,0.06049066781997681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,1,0.062261333068211876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,3,0.06163733204205831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,1,128,1,float16,fp8,63,0.03860266755024592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,3,0.06193066636721293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,7,0.06107200185457865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,7,0.061381335059801735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,15,0.06086933116118113
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,15,0.06217599908510844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,31,0.06582933167616527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,31,0.06234133243560791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,63,0.06921066840489705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,63,0.06623999774456024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,127,0.07538666824499766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,127,0.08449600140253703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,float16,255,0.1122773289680481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,2,128,1,float16,fp8,255,0.10332266489664714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,1,0.11451199650764465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,1,0.11339199542999268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,3,0.11387200156847636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,3,0.11270399888356526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,7,0.11504000425338745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,15,0.12231999635696411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,31,0.1257866621017456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,15,0.11375466982523601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,63,0.1267466644446055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,7,0.11205333471298218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,63,0.1243839959303538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,127,0.13568533460299173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,127,0.1567359964052836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,31,0.12126933534940083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,float16,255,0.20221332708994547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,4,128,1,float16,fp8,255,0.18884267409642538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,1,0.20859734217325845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,1,0.20332266887029013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,3,0.21145600080490112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,3,0.20454933245976767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,7,0.22290666898091635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,7,0.20700265963872275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,31,0.22711465756098428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,31,0.21913599967956543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,63,0.23100799322128296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,63,0.22232532501220703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,15,0.21955732504526773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,15,0.22789333264033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,127,0.24422933657964072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,1,0.06464533507823944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,1,0.06544533371925354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,127,0.2909653385480245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,fp8,255,0.341594656308492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,16,8,128,1,float16,float16,255,0.3744266827901204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,3,0.06529066463311513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,3,0.06530133386452992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,7,0.06427200138568878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,7,0.06509333352247874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,15,0.06710933148860931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,63,0.07419200241565704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,63,0.07629866898059845
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,15,0.06592533489068349
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,31,0.07247466842333476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,31,0.06649599969387054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,float16,127,0.08022400240103404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,1,0.11126933495203654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,1,0.1125866671403249
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,1,128,1,float16,fp8,127,0.0902239978313446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,3,0.11128532886505127
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,3,0.11267733573913574
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,7,0.11089066664377849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,7,0.11117866635322571
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,15,0.11705600221951802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,15,0.11265066266059875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,31,0.12225600083669026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,31,0.11869866649309795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,63,0.12482666969299316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,63,0.12377066413561504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,float16,127,0.13530133167902628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,2,128,1,float16,fp8,127,0.15176000197728476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,1,0.21591466665267944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,1,0.212826669216156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,3,0.22472000122070312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,3,0.2130026618639628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,7,0.2351413369178772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,7,0.21849066019058228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,15,0.2354080080986023
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,15,0.22890132665634155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,31,0.23553599913915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,31,0.23167999585469565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,63,0.24156800905863443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,63,0.2330026626586914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,float16,127,0.25618666410446167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,1,0.4198880195617676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,1,0.39474133650461835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,4,128,1,float16,fp8,127,0.30187199513117474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,3,0.4382079839706421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,7,0.445306658744812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,7,0.42182934284210205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,15,0.4431519905726115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,3,0.40777067343393963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,15,0.4259893496831258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,31,0.44042134284973145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1,0.011578666667143503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,3,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,63,0.44401601950327557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,3,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,1,0.012335999558369318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,31,0.42765335241953534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,7,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,15,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,31,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,63,0.011605333536863327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,31,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,63,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,127,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,7,0.011349332829316458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,15,0.012522666404644648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,127,0.013327999661366144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,255,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,255,0.01434133326013883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,511,0.014682666709025701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,511,0.01725333308180173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,1023,0.019088000059127808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,63,0.436245322227478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,1023,0.021253332495689392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,2047,0.027674667537212372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,4095,0.05037866532802582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,8191,0.08182399968306224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,fp8,127,0.5622239907582601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,2047,0.030981334547201794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,4095,0.04655466477076212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,16383,0.15042666594187418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,16383,0.1827359994252523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,8191,0.09548800190289815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,float16,32767,0.2877013285954793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1,0.011717333147923151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,16,8,128,1,float16,float16,127,0.4756106535593669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,1,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,3,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,3,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,1,128,1,float16,fp8,32767,0.3505760033925374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,7,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,7,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,15,0.011530666301647821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,15,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,31,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,63,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,63,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,127,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,127,0.014277332772811254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,255,0.013914667069911957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,31,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,255,0.014949332922697067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,511,0.015253332753976187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,511,0.016901332885026932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,1023,0.019648000597953796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,1023,0.021759999295075733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,2047,0.02874133239189784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,2047,0.031311998764673867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,16383,0.1721120079358419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,4095,0.047600001096725464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,32767,0.3243040045102437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,4095,0.05286400020122528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,8191,0.08382933338483174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,fp8,8191,0.09341333309809367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,16383,0.15187199910481772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,7,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,2,128,1,float16,float16,32767,0.2892426649729411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,1,0.01310933381319046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,3,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,3,0.013280000537633896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,7,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,15,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,15,0.013077333569526672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,31,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,31,0.013007999708255133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,63,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,63,0.012752000242471695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,127,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,127,0.014309333016475042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,1023,0.02197333425283432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,255,0.013744000345468521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,255,0.01544533297419548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,511,0.01589866727590561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,4095,0.057034666339556374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,511,0.017765333255132038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,1023,0.021162666380405426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,2047,0.03660800059636434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,2047,0.03258133431275686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,4095,0.0572320024172465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,8191,0.10805867115656535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,8191,0.0981333355108897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,16383,0.19417067368825278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,16383,0.18367467323939005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,float16,32767,0.3871946732203166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,4,128,1,float16,fp8,32767,0.35040533542633057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,1,0.013104000439246496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,15,0.013466666142145792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,3,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,3,0.013007999708255133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,31,0.013610667238632837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,63,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,7,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,63,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,7,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,15,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,31,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,127,0.013082666943470636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,127,0.014736000448465347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,255,0.01413333291808764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,1023,0.03515733281771342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,255,0.015194666882356008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,511,0.017173333714405697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,511,0.01841066653529803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,1023,0.024154665569464367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,2047,0.055248002211252846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,2047,0.03977066775163015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,4095,0.09899733463923137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,4095,0.06294399996598561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,8191,0.18289599816004434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,8191,0.1090773344039917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,16383,0.3534880081812541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,16383,0.1996799906094869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,float16,32767,0.6896266937255859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,1,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,7,0.012485332787036896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,15,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,16,8,128,1,float16,fp8,32767,0.3802880048751831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,3,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,3,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,7,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,63,0.012736000120639801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,15,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,31,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,31,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,63,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,511,0.015141333142916361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,127,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,127,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,1023,0.021935999393463135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,255,0.013770667215188345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,255,0.015130666395028433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,511,0.01703466723362605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,1023,0.019839999576409657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,2047,0.030048000315825146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,2047,0.03200000027815501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,4095,0.04924799998601278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,4095,0.05232533315817515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,8191,0.0848479966322581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,8191,0.09943999846776326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,3,0.012304000556468964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,float16,16383,0.1555466651916504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,1,128,1,float16,fp8,16383,0.1816640098889669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,1,0.012746666868527731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,3,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,7,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,7,0.012773333738247553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,15,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,15,0.012725333372751871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,31,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,31,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,63,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,63,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,127,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,127,0.014661333213249842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,255,0.014074667046467463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,255,0.014805333067973455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,511,0.015562667200962702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,511,0.017637333522240322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,1023,0.023552000522613525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,1023,0.022122666239738464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,2047,0.036890665690104164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,2047,0.0339626669883728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,4095,0.06035733222961426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,4095,0.05702400207519531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,8191,0.10796800255775452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,8191,0.09856533010800679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,float16,16383,0.18668800592422485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,2,128,1,float16,fp8,16383,0.1833546757698059
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,1,0.01349866638580958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,3,0.01227733368674914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,3,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,7,0.01227733368674914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,7,0.013424000392357508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,15,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,15,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,127,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,31,0.012495999534924826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,31,0.01349866638580958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,63,0.012634667257467905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,63,0.013280000537633896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,127,0.014842666685581207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,255,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,255,0.015557333827018738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,511,0.017456000049908955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,511,0.018373332917690277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,1023,0.035045333206653595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,1023,0.02518933266401291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,4095,0.0634986658891042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,2047,0.0562666654586792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,2047,0.04001066585381826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,8191,0.11085866888364156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,4095,0.0977226694424947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,1,0.012602667013804117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,16383,0.3529333273569743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,float16,8191,0.18269334236780801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,12,4,128,1,float16,fp8,16383,0.20048532883326212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,3,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,3,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,7,0.011050666371981302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,7,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,15,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,15,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,31,0.01102399950226148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,31,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,255,0.014101333916187286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,63,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,63,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,127,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,127,0.014085333794355392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,255,0.013397333522637686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,2047,0.015802666544914246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,511,0.013189333180586496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,511,0.014416000495354334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,1023,0.013525333255529404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,1023,0.014469332993030548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,16383,0.02219199885924657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,2047,0.014479999740918478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,4095,0.015850666910409927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,4095,0.016666666915019352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,8191,0.017829333742459614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,8191,0.01904533306757609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,16383,0.024517332514127094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,1,0.012624000509579977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,32767,0.03216533362865448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,32767,0.034714666505654655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,float16,65535,0.05116266508897146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,1,128,1,float16,fp8,65535,0.055493334929148354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,15,0.012576000144084295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,3,0.011045332998037338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,3,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,7,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,63,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,127,0.01110400011142095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,7,0.011978667229413986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,15,0.01099733387430509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,31,0.011247999966144562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,31,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,511,0.01463466634353002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,63,0.010853332777818045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,127,0.014368000129858652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,255,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,255,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,511,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,1023,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,1023,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,2047,0.014794666320085526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,2047,0.016186666985352833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,16383,0.02550933261712392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,32767,0.03393599887688955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,4095,0.016255999604860943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,65535,0.0537013312180837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,4095,0.017050666113694508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,8191,0.019178666174411774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,8191,0.019952000429232914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,float16,16383,0.02312533309062322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,32767,0.03544000039498011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,2,128,1,float16,fp8,65535,0.05783999959627787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1,0.01129066695769628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,1,0.012458667159080505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,3,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,3,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,7,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,63,0.011312000453472137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,7,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,15,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,15,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,31,0.011194666226704916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,31,0.012719999998807907
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,63,0.012096000214417776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,127,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,127,0.01434133326013883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,1023,0.01540800059835116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,255,0.013466666142145792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,255,0.014442666123310724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,511,0.013482666263977686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,4095,0.0174346665541331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,511,0.015109332899252573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,1023,0.014485333114862442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,2047,0.01613866661985715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,2047,0.016538667182127636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,4095,0.016885332763195038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,8191,0.018842666099468868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,8191,0.020197333147128422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,16383,0.02447466552257538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,16383,0.025610665480295818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1,0.010874666273593903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,32767,0.0383146678407987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,32767,0.0377866675456365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,float16,65535,0.0598826656738917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,12,4,128,1,float16,fp8,65535,0.06229333579540253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,1,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,3,0.010821333775917688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,15,0.01250133290886879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,31,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,3,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,31,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,7,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,7,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,15,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,63,0.010821333775917688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,63,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,127,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,127,0.014245333770910898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,1023,0.01532799998919169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,255,0.013386666774749756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,255,0.0145066666106383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,511,0.013327999661366144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,511,0.014240000396966934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,8191,0.0215786670645078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,1023,0.014533333480358124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,16383,0.029653333127498627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,2047,0.01588800052801768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,2047,0.016442666451136272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,4095,0.01802666609485944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,4095,0.01916266605257988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,8191,0.02382933348417282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,1,0.011920000116030375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,16383,0.034058667719364166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,32767,0.04866133133570353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,3,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,32767,0.05243200063705444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,float16,65535,0.08388266960779826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,1,128,1,float16,fp8,65535,0.09379733602205913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,15,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,3,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,7,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,63,0.012410666793584824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,7,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,15,0.011221333096424738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,31,0.011253333340088526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,31,0.012453333785136541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,63,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,127,0.011301333705584208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,127,0.014469332993030548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,255,0.013552000125249227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,255,0.014378666877746582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,511,0.013450667262077332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,511,0.015178666760524115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,4095,0.01989866668979327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,1023,0.01461333284775416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,1023,0.01624533285697301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,2047,0.016821333517630894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,2047,0.017184000462293625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,4095,0.019296000401178997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,8191,0.022218666970729828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,8191,0.024351999163627625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,16383,0.03086400032043457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,16383,0.03446933378775915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,32767,0.04993066688378652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,32767,0.056373332937558494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,float16,65535,0.08776000142097473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,2,128,1,float16,fp8,65535,0.10218666990598042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,1,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,3,0.011066666493813196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,31,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,3,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,7,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,63,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,7,0.012186666329701742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,15,0.011658667276302973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,15,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,255,0.015146666516860327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,31,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,63,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,127,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,127,0.014186666657527288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,2047,0.016751999656359356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,255,0.013541333377361298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,511,0.014815999815861383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,511,0.015392000476519266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,8191,0.023749334116776783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,1023,0.016165333489576977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,1023,0.016986666868130367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,2047,0.01735466718673706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,4095,0.01878400022784869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,4095,0.020143999407688778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,65535,0.10374400019645691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,8191,0.02476799984773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,16383,0.03829866647720337
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,16383,0.03644266724586487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,float16,32767,0.06224533418814341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,32767,0.059546664357185364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,12,4,128,1,float16,fp8,65535,0.10098666946093242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1,0.012784000486135483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,1,0.013397333522637686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,3,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,3,0.013493333011865616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,7,0.012901333471139273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,7,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,15,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,15,0.013381333400805792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,31,0.012789333860079447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,31,0.013663999736309052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,63,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,63,0.013631999492645264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,255,0.015626666446526844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,511,0.016255999604860943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,1023,0.02142400046189626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,1023,0.02276266614596049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,127,0.013194666554530462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,127,0.014815999815861383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,255,0.014511999984582266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,511,0.01826133330663045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,2047,0.037818667789300285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,2047,0.03286399940649668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,1,0.014362666755914688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,4095,0.07108266651630402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,4095,0.05835733314355215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,float16,8191,0.10965333382288615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,7,0.014453332871198654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,1,128,1,float16,fp8,8191,0.09965333342552185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,15,0.013093333691358566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,3,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,3,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,7,0.012901333471139273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,31,0.01301866645614306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,15,0.01393066719174385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,31,0.014256000518798828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,255,0.014789332946141561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,63,0.012768000364303589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,63,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,511,0.01736533393462499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,127,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,127,0.015279999623696009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,255,0.016037333756685257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,511,0.01907733331123988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,1023,0.035402665535608925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,4095,0.06404800216356914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,1023,0.025061334172884624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,2047,0.05678399900595347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,2047,0.041152000427246094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,4095,0.0990613301595052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,float16,8191,0.1823520064353943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,2,128,1,float16,fp8,8191,0.1146453320980072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1,0.016575999557971954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,7,0.018138666947682697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,1,0.01820266619324684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,3,0.01695999999841054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,3,0.01800000046690305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,31,0.017935999979575474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,7,0.01708799973130226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,15,0.01703466723362605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,15,0.018378666291634243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,31,0.01692266638080279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,255,0.021157334248224895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,63,0.01695999999841054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,63,0.017770666629076004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,127,0.017338667064905167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,127,0.020879998803138733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,255,0.022330666581789654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,511,0.03749866783618927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,511,0.02807466685771942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,1023,0.05995733539263407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,1023,0.04397333165009817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,2047,0.10048000017801921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,2047,0.06781333188215892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,4095,0.18478933970133463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,4095,0.1151146690050761
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,1,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1,0.01116266722480456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,3,0.010773333410422007
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,float16,8191,0.35234665870666504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,12,4,128,1,float16,fp8,8191,0.2069279948870341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,31,0.010826667149861654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,3,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,7,0.010922666639089584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,7,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,15,0.010847999403874079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,15,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,31,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,255,0.014181333283583323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,63,0.010805333654085795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,63,0.011653333902359009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,127,0.011055999745925268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,127,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,255,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,511,0.014181333283583323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,511,0.015599999576807022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,1023,0.015642666568358738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,1023,0.016336000214020412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,2047,0.016794666647911072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,2047,0.018757333358128864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,16383,0.04827199876308441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,4095,0.021216000119845074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,4095,0.023498666783173878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,32767,0.0976586639881134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,8191,0.029674666623274486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,8191,0.03306666761636734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,16383,0.05219733218352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,32767,0.08371733625729878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,3,0.010922666639089584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,float16,65535,0.15294933319091797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1,0.010885333021481832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,7,0.011061333119869232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,1,128,1,float16,fp8,65535,0.18408000469207764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,1,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,3,0.011711999773979187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,7,0.011770666887362799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,15,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,15,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,127,0.014378666877746582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,31,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,31,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,63,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,63,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,127,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,255,0.013562666873137156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,255,0.014671999961137772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,511,0.014661333213249842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,511,0.01621333385507266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,4095,0.023914667467276256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,1023,0.016501333564519882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,1023,0.016666666915019352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,2047,0.017498667041460674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,16383,0.05464000006516775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,2047,0.019391999890406925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,4095,0.02197333425283432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,8191,0.030405332644780476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,8191,0.03390933324893316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1,0.011503999431928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,16383,0.05005866785844167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,32767,0.0846720039844513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,3,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,32767,0.10156800349553426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,float16,65535,0.1555359959602356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,2,128,1,float16,fp8,65535,0.18284799655278525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,1,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,3,0.011503999431928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,7,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,7,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,63,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,15,0.011519999553759893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,15,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,255,0.014149333039919535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,31,0.011594666788975397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,31,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,511,0.016314666718244553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,63,0.011365332951148352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,127,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,127,0.014885333677132925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,255,0.014896000425020853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,511,0.01565333331624667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,1023,0.016682667036851246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,1023,0.016810666769742966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,2047,0.017653333644072216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,2047,0.01947733387351036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,4095,0.022298666338125866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,4095,0.024282666544119518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,8191,0.03759466608365377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,8191,0.03706666578849157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,16383,0.05952000121275584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,16383,0.05870933334032694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,32767,0.10335466265678406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,32767,0.10078932841618855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,float16,65535,0.20265599091847739
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,3,0.014111999422311783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,12,4,128,1,float16,fp8,65535,0.1829493244489034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1,0.014256000518798828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,1,0.015061333775520325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,3,0.014864000181357065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,7,0.0141546664138635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,7,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,15,0.014159999787807465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,15,0.015311999867359797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,31,0.014005333185195923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,31,0.015247999380032221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,63,0.014127999544143677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,63,0.014997333288192749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,127,0.014645333091417948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,511,0.0194560003777345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,127,0.016586666305859882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,255,0.01618133361140887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,255,0.01740266631046931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,511,0.020047999918460846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,1023,0.03738133360942205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,1023,0.0268053337931633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,2047,0.05860800047715505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,2047,0.042954668402671814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1,0.017557332913080852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,float16,4095,0.10026133060455322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,1,0.018960000326236088
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,1,128,1,float16,fp8,4095,0.06733866532643636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,3,0.01747200017174085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,3,0.018570666511853535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,7,0.017525333911180496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,7,0.019039999693632126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,15,0.018863999595244724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,15,0.017711999515692394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,31,0.01870399961868922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,63,0.01762666677435239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,31,0.0174346665541331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,63,0.018794666975736618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,127,0.01815466706951459
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,127,0.02189333240191142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,255,0.02146666745344798
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,255,0.023130667706330616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,511,0.04010133445262909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,1023,0.060736000537872314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,511,0.02985599885384242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,1023,0.04626133541266123
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,4095,0.18528000513712564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,3,0.02611200014750163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,1,0.02605866640806198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,float16,2047,0.10195199648539226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,4095,0.11660266915957133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,2,128,1,float16,fp8,2047,0.07010666529337566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1,0.024106666445732117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,7,0.025754667818546295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,3,0.024314666787783306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,7,0.026000000536441803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,15,0.024346667031447094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,63,0.02475733309984207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,63,0.025936000049114227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,15,0.025978667040665943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,31,0.024325333535671234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,31,0.026133333643277485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,255,0.04327466587225596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,127,0.026389333109060924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,127,0.03071466585000356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,255,0.03425066669782003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,511,0.06472533444563548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,511,0.0525439977645874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,1023,0.10391466816266377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,2047,0.18639467159907022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,1023,0.0745600014925003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,2047,0.12258666753768921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1,0.01959466685851415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,fp8,4095,0.2106026609738668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,12,4,128,1,float16,float16,4095,0.3524213234583537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,1,0.021349333226680756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,3,0.022629333039124806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,3,0.019551999866962433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,15,0.019706666469573975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,7,0.02161066730817159
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,7,0.019695999721686046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,15,0.021205333371957142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,63,0.021157334248224895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,31,0.019866666446129482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,31,0.021349333226680756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,63,0.019871999820073444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,127,0.020362666497627895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,127,0.024400000770886738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,255,0.02624000112215678
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,255,0.025626666843891144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,511,0.042405332128206887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,511,0.033439998825391136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,float16,1023,0.06398933132489522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,1,128,1,float16,fp8,1023,0.04974933465321859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1,0.026250667870044708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,1,0.028351999819278717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,3,0.02587199956178665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,3,0.027637332677841187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,7,0.025936000049114227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,7,0.02794133375088374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,15,0.02625600000222524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,15,0.028218666712443035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,31,0.02632533262173335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,31,0.02827200045188268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,63,0.027877333263556164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,63,0.026309333741664886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,127,0.028330666323502857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,127,0.032645332316557564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,255,0.045519997676213585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,255,0.03997333347797394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,511,0.06685333450635274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1,0.038949333131313324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,1,0.04223466912905375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,511,0.05530133346716563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,float16,1023,0.10750933488210042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,3,0.03937066594759623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,3,0.04240000247955322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,7,0.04177600145339966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,7,0.0391893337170283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,2,128,1,float16,fp8,1023,0.0780266672372818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,15,0.039135999977588654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,15,0.04225599765777588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,31,0.042208001017570496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,63,0.041519999504089355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,31,0.039093332986036934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,63,0.041802664597829185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,127,0.04807466765244802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,255,0.06948799888292949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,255,0.06628799935181935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,127,0.051455999414126076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,511,0.11200533310572307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,511,0.08910399675369263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,1,0.033770665526390076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,1,0.035642666121323906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,3,0.03350399931271871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,float16,1023,0.19171200195948282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,12,4,128,1,float16,fp8,1023,0.1332533359527588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,7,0.03340800106525421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,15,0.03365333378314972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,3,0.035546667873859406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,7,0.03524799893299738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,15,0.03595199932654699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,31,0.03340800106525421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,63,0.03570133447647095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,127,0.03716800113519033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,127,0.03950933367013931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,255,0.05332799752553304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,255,0.04574400186538696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,31,0.03519999980926514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,63,0.03420799970626831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,float16,511,0.07390933235486348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,1,0.04527466495831808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,1,128,1,float16,fp8,511,0.06376533210277557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,1,0.04808533191680908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,3,0.04540266593297323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,3,0.04842666784922282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,7,0.045253331462542214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,7,0.04822933177153269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,15,0.04799466828505198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,15,0.04560000201066335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,31,0.048170665899912514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,31,0.04582933088143667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,63,0.047685335079828896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,63,0.04853333532810211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,127,0.05816000203291575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,127,0.06107733150323232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,255,0.07815999786059062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,255,0.0738453318675359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,1,0.07130666573842366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,float16,511,0.11986666917800903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,2,128,1,float16,fp8,511,0.09857066472371419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,1,0.07745066781838734
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,3,0.07147733370463054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,3,0.07681066791216533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,7,0.07149333258469899
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,7,0.07670933504899342
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,15,0.07186666627724965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,31,0.07747733096281688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,63,0.07918933530648549
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,63,0.0800853321949641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,15,0.07704000174999237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,127,0.0863146682580312
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,31,0.07494933406511943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1,0.01137599969903628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,127,0.09910933176676433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,3,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,3,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,1,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,7,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,15,0.01099733387430509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,7,0.011071999867757162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,15,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,255,0.12069333593050639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,31,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,255,0.11796266833941142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,31,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,63,0.011055999745925268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,63,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,255,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,float16,511,0.2035306692123413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,12,4,128,1,float16,fp8,511,0.16595733165740967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,127,0.013306666165590286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,127,0.011045332998037338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,255,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,511,0.014789332946141561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,511,0.01634666696190834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,1023,0.016783999900023144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,2047,0.02362666775782903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,4095,0.02938666691382726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,4095,0.03365866591533025
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,8191,0.04785066843032837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,8191,0.05267733335494995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,1023,0.01907733331123988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,16383,0.08434666196505229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,2047,0.021216000119845074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,16383,0.09757333000500996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,32767,0.1516800026098887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,32767,0.1867093245188395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,3,0.013445333888133367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,float16,65535,0.2889973322550456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,1,128,1,float16,fp8,65535,0.3527146577835083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1,0.011247999966144562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,1,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,3,0.011519999553759893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,7,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,63,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,7,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,15,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,15,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,31,0.011413333316644033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,31,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,63,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,511,0.015557333827018738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,127,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,511,0.01666133354107539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,127,0.013749333719412485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,255,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,255,0.01624533285697301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,1023,0.01754666616519292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,1023,0.019882666567961376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,2047,0.02162666618824005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,2047,0.024149333437283833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,4095,0.031471999982992806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,4095,0.034234667817751564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,8191,0.05133333305517832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,8191,0.05515733361244202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,16383,0.08471999565760295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,16383,0.10181867082913716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,32767,0.1830186645189921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,32767,0.15542933344841003
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,float16,65535,0.29872532685597736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,7,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,2,128,1,float16,fp8,65535,0.35229865709940594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,1,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,3,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,3,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,7,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,15,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,15,0.012885333349307379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,31,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,31,0.01302933320403099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,63,0.011829332758982977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,511,0.01634666696190834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,63,0.012837332983811697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,127,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,127,0.014186666657527288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,255,0.013674666484196981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,2047,0.024154665569464367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,255,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,511,0.01735466718673706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,1023,0.017765333255132038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,1023,0.019226666539907455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,2047,0.02351466566324234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,4095,0.043290664752324425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,4095,0.03458133339881897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,8191,0.0618399977684021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,32767,0.1834239959716797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,8191,0.05946666498978933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,16383,0.12828266620635986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,16383,0.10025599598884583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,32767,0.20408000548680624
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,float16,65535,0.46595199902852374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,12,4,128,1,float16,fp8,65535,0.3511199951171875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,1,0.03746666759252548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,1,0.038133333126703896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,3,0.037946666280428566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,3,0.038021333515644073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,7,0.03728000074625015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,7,0.037861332297325134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,15,0.037765334049860634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,15,0.038373333712418876
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,31,0.03741333385308584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,31,0.038362666964530945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,63,0.04191466669241587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,127,0.04606399933497111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,63,0.03860799968242645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,127,0.04952000081539154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,float16,255,0.06566933294137318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,1,128,1,float16,fp8,255,0.06328000128269196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,1,0.06328533093134563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,1,0.06208533545335134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,3,0.06354133288065593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,3,0.06195199986298879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,7,0.06177600224812826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,7,0.06414400041103363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,15,0.06390933195749919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,15,0.06226666768391927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,31,0.06811733543872833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,31,0.06250666578610738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,63,0.06750399867693584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,63,0.07096533477306366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,127,0.07537599901358287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,float16,255,0.1122773289680481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,127,0.08787199854850769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,2,128,1,float16,fp8,255,0.10550399621327718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,1,0.11396800478299458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,1,0.11226666967074077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,3,0.11382933457692464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,3,0.11262399951616923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,7,0.11406933267911275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,31,0.12495467066764832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,31,0.119759996732076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,7,0.113237331310908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,15,0.12302933136622111
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,15,0.11272000273068745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,63,0.12613866726557413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,63,0.12429333726565044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,127,0.13414399822553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,float16,255,0.20127999782562256
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,255,0.1885813275973002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,1,0.06410133341948192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,1,0.06517866750558217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,12,4,128,1,float16,fp8,127,0.15680000185966492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,3,0.06472533444563548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,3,0.06462933123111725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,7,0.06528533498446147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,7,0.06465066472689311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,15,0.06566933294137318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,15,0.06586666901906331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,31,0.07124799986680348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,31,0.0652106652657191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,63,0.07334933181603749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,63,0.0741066684325536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,float16,127,0.07915199796358745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,1,128,1,float16,fp8,127,0.08917333682378133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,1,0.11614933609962463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,1,0.11220799883206685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,3,0.11434132854143779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,3,0.11488533020019531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,7,0.1172106663386027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,7,0.11140799522399902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,15,0.12618133425712585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,15,0.11242666840553284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,31,0.12398399909337361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,31,0.1279253363609314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,63,0.12769066294034323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,63,0.12593600153923035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,fp8,127,0.15692266821861267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,2,128,1,float16,float16,127,0.13749333222707114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,1,0.2189813256263733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,1,0.21220266819000244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,3,0.2195039987564087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,3,0.2140586574872335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,7,0.23254400491714478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,7,0.21341866254806519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,31,0.2384480039278666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,31,0.2305013338724772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,15,0.23426665862401327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,15,0.2283626596132914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,3,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1,0.011312000453472137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,1,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,3,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,7,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,7,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,15,0.01146666705608368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,15,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,31,0.011488000551859537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,31,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,63,0.23971732457478842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,63,0.011626667032639185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,63,0.012229333321253458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,127,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,127,0.013248000293970108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,255,0.014277332772811254
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,511,0.01492799942692121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,255,0.012890666723251343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,511,0.016613333175579708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,1023,0.019152000546455383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,1023,0.021530665457248688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,63,0.2334346572558085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,2047,0.027823999524116516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,2047,0.03134933362404505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,4095,0.04537599782148997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,4095,0.050885334610939026
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,8191,0.08212266862392426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,16383,0.18321067094802856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,8191,0.09345066547393799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,float16,127,0.2553173303604126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,16383,0.15158933401107788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,float16,32767,0.2871146599451701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,1,128,1,float16,fp8,32767,0.3537333408991496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1,0.01179733375708262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,1,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,3,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,3,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,12,4,128,1,float16,fp8,127,0.2975520094235738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,7,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,15,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,15,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,7,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,31,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,31,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,63,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,127,0.011770666887362799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,127,0.013978666315476099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,255,0.013679999858140945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,255,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,63,0.012794667234023413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,2047,0.02903999884923299
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,511,0.015376000354687372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,511,0.017498667041460674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,1023,0.019786667078733444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,8191,0.0844586690266927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,8191,0.09930666287740071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,1023,0.02183466653029124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,16383,0.15280532836914062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,2047,0.031914666295051575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,4095,0.04923733572165171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,float16,32767,0.2929706573486328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,4095,0.0536106675863266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,1,0.013301332791646322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,3,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,3,0.01314666618903478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,16383,0.1837386687596639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,2,128,1,float16,fp8,32767,0.34594666957855225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,31,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,7,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,7,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,15,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,15,0.01394133393963178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,31,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,63,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,63,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,127,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,127,0.014730667074521383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,255,0.013839999834696451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,255,0.014901333798964819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,511,0.015754666179418564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,511,0.017322666943073273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,1023,0.021301334102948506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,1023,0.022341333329677582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,2047,0.035989334185918175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,2047,0.03453333427508672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,4095,0.07029333213965099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,4095,0.056799997886021934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,8191,0.10638399918874104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,8191,0.09788266817728679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,16383,0.18598934014638266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,16383,0.18373332420984903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,float16,32767,0.4618186553319295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,12,4,128,1,float16,fp8,32767,0.3476746479670207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,1,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,3,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,3,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,31,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,7,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,63,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,63,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,7,0.012522666404644648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,15,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,15,0.012373333175977072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,31,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,127,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,127,0.014074667046467463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,255,0.013232000172138214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,255,0.015013333410024643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,511,0.016458666572968166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,511,0.0170666662355264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,1023,0.01950399950146675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,1023,0.021520001192887623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,2047,0.02890666574239731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,8191,0.09289066990216573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,2047,0.03197333216667175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,4095,0.04909333089987437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,4095,0.05259199937184652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,32767,0.29013333717981976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,32767,0.325546662012736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,8191,0.08565333485603333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,3,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,3,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,float16,16383,0.15125333269437155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,1,128,1,float16,fp8,16383,0.17130666971206665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,1,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,7,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,7,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,15,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,15,0.012778667112191519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,31,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,31,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,63,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,63,0.012805332740147909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,127,0.012421333541472753
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,127,0.014192000031471252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,255,0.013967999567588171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,255,0.014848000059525171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,511,0.016063999384641647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,511,0.017386666188637417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,1023,0.021338666478792827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,1023,0.021935999393463135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,2047,0.035232000052928925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,2047,0.033941333492596946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,4095,0.07030400137106578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,4095,0.05675200124581655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,8191,0.1074079970518748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,8191,0.0990826686223348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,16383,0.1867146690686544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,16383,0.18054399887720743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1,0.012432000289360682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,float16,32767,0.35891199111938477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,2,128,1,float16,fp8,32767,0.3527626593907674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,1,0.013424000392357508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,3,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,3,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,7,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,31,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,7,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,15,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,15,0.013381333400805792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,31,0.013258667041858038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,63,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,63,0.013440000514189402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,127,0.012709333250919977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,127,0.014709333578745524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,255,0.014666666587193808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,255,0.015504000087579092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,511,0.016970666746298473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,511,0.017984000345071156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,1023,0.03510933369398117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,1023,0.0236160010099411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,2047,0.05676800012588501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,2047,0.039813332259655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,4095,0.09807466467221577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,4095,0.06368533273537953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,8191,0.1834239959716797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,16383,0.35304534435272217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,8191,0.10986133416493733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,16383,0.2008906602859497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1,0.016421332955360413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,1,0.017349333812793095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,float16,32767,0.6920159657796224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,4,128,1,float16,fp8,32767,0.382421334584554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,3,0.016586666305859882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,3,0.017562666287024815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,7,0.016496000190575916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,7,0.017450666675964992
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,15,0.016336000214020412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,15,0.01735466718673706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,31,0.016517333686351776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,127,0.020080000162124634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,31,0.0176959993938605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,63,0.016677333662907284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,511,0.037317333122094475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,511,0.026885333160559338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,63,0.017594666530688603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,127,0.017029333859682083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,255,0.02022933339079221
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,255,0.02160000056028366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,1023,0.058821335434913635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,1023,0.04301333427429199
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,2047,0.0993386705716451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,2047,0.06675200164318085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,4095,0.18375466267267862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,4095,0.11196266611417134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,8191,0.3503199815750122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,8191,0.20350933074951172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,16383,0.6875893274943033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,16383,0.38361068566640216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1,0.010992000500361124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,1,0.011850666254758835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,3,0.010970667004585266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,fp8,32767,0.7422080039978027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,8,8,128,1,float16,float16,32767,1.4807573954264324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,31,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,3,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,7,0.011221333096424738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,7,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,15,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,15,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,255,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,31,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,511,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,63,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,63,0.011018666128317514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,127,0.010901333143313726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,127,0.012986666212479273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,255,0.01309866706530253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,511,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,4095,0.016058667252461117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,1023,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,1023,0.013621332744757334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,2047,0.013653332988421122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,16383,0.024080000817775726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,2047,0.0144213338692983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,4095,0.015008000036080679
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,8191,0.017322666943073273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,8191,0.018826667219400406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,131071,0.08764266967773438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,16383,0.02218666672706604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,32767,0.03155199935038885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,32767,0.034261333445707955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,float16,65535,0.04975999891757965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,65535,0.05498133103052775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,1,128,1,float16,fp8,131071,0.0978559950987498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,1,0.011994666109482447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,3,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,31,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,3,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,7,0.010832000523805618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,7,0.012495999534924826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,15,0.01110400011142095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,15,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,31,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,63,0.011301333705584208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,255,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,63,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,127,0.011173332730929056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,127,0.013104000439246496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,255,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,511,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,511,0.013834666460752487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,4095,0.015765332927306492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,1023,0.012896000097195307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,1023,0.014618666221698126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,8191,0.019541333119074505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,2047,0.014015999933083853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,2047,0.015749332805474598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,32767,0.03301866600910822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,4095,0.0170666662355264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,8191,0.018346666047970455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,131071,0.09113599856694539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,131071,0.10620266199111938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,16383,0.02239466706911723
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,16383,0.025445332129796345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,32767,0.03504000107447306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,float16,65535,0.05303466816743215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,2,128,1,float16,fp8,65535,0.057914664347966514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,15,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,1,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,31,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,3,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,3,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,63,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,7,0.011391999820868174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,7,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,255,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,15,0.012330666184425354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,31,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,63,0.011034666250149408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,127,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,127,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,255,0.013493333011865616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,511,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,511,0.01458666721979777
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,1023,0.014848000059525171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,1023,0.013429333766301474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,2047,0.014965333044528961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,2047,0.016336000214020412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,4095,0.016490666816631954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,4095,0.0170666662355264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,8191,0.01860800012946129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,8191,0.019866666446129482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,16383,0.02348266790310542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,16383,0.02532266577084859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,32767,0.03833066672086716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,32767,0.03689600030581156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,1,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,3,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,65535,0.05884266893068949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,65535,0.06165333092212677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,float16,131071,0.13637333114941916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,4,128,1,float16,fp8,131071,0.10652800401051839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1,0.011424000064531961
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,3,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,7,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,63,0.011514666179815928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,7,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,15,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,15,0.012624000509579977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,255,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,31,0.01138666644692421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,31,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,63,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,1023,0.0138026662170887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,127,0.011301333705584208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,127,0.01332266628742218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,255,0.014384000251690546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,511,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,511,0.014757333944241205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,1023,0.015119999647140503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,2047,0.01525866612792015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,16383,0.03766933331886927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,2047,0.016437333077192307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,4095,0.016623999923467636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,4095,0.01749333366751671
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,8191,0.019797333826621372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,8191,0.02006400004029274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,16383,0.02619733413060506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,32767,0.060565332571665444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,131071,0.13037866353988647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,32767,0.046629334489504494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,65535,0.10385066270828247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,fp8,65535,0.07408533493677776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,8,8,128,1,float16,float16,131071,0.19106666247049967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1,0.010885333021481832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,15,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,1,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,3,0.010970667004585266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,3,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,7,0.010890666395425797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,7,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,15,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,31,0.011066666493813196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,31,0.011717333147923151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,63,0.011141333729028702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,63,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,511,0.012517333030700684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,127,0.010837333897749582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,127,0.012714666624863943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,255,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,255,0.013338666409254074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,511,0.013424000392357508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,1023,0.013834666460752487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,4095,0.018453333526849747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,1023,0.014240000396966934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,2047,0.014655999839305878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,2047,0.015957333147525787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,4095,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,8191,0.02027200038234393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,8191,0.023007998863856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,16383,0.028991999725500744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,32767,0.04695466657479604
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,16383,0.03265066693226496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,32767,0.05179200073083242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,1,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,65535,0.0824480007092158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,65535,0.0921493371327718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,float16,131071,0.1498026649157206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,1,128,1,float16,fp8,131071,0.17249600092569986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,3,0.011077333241701126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,3,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,7,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,7,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,15,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,15,0.012015999605258306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,31,0.01129066695769628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,31,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,63,0.01128000020980835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,63,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,127,0.011055999745925268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,127,0.013445333888133367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,255,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,2047,0.015791999797026317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,255,0.013514666507641474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,4095,0.018415999909241993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,4095,0.019088000059127808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,511,0.012703999876976013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,511,0.014384000251690546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,16383,0.030266667405764263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,1023,0.01369599997997284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,1023,0.015541333705186844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,2047,0.016522667060295742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,8191,0.021007999777793884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,8191,0.023823998868465424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,16383,0.033813332517941795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,32767,0.048528000712394714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1,0.011461333682139715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,1,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,32767,0.054287999868392944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,65535,0.0841919978459676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,65535,0.09959999720255534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,7,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,float16,131071,0.15403733650843301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,2,128,1,float16,fp8,131071,0.18211199839909872
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,3,0.011440000186363855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,3,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,7,0.011365332951148352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,15,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,15,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,127,0.013487999637921652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,31,0.011274666835864386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,31,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,63,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,63,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,127,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,255,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,2047,0.01640533283352852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,255,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,511,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,4095,0.019365333020687103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,511,0.014384000251690546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,1023,0.015040000279744467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,16383,0.036746665835380554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,1023,0.015967999895413715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,2047,0.016389333953460056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,4095,0.018042666216691334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,8191,0.022815999885400135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,8191,0.023738667368888855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,16383,0.03619199991226196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,32767,0.06204266846179962
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,32767,0.05869866907596588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,65535,0.10204266508420308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,3,0.011488000551859537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,65535,0.10076799988746643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,float16,131071,0.2037013371785482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,4,128,1,float16,fp8,131071,0.18176533778508505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1,0.011578666667143503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,1,0.012432000289360682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,15,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,3,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,7,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,7,0.012613333761692047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,15,0.011557333171367645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,31,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,127,0.014032000054915747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,31,0.012416000167528788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,63,0.011450666934251785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,511,0.013829333086808523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,63,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,127,0.011413333316644033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,255,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,255,0.014501333236694336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,511,0.014922666052977243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,1023,0.015301333119471868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,4095,0.019648000597953796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,1023,0.016282666474580765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,2047,0.01646399994691213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,2047,0.01708799973130226
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,4095,0.019146667172511418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,8191,0.0364479993780454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,16383,0.05793599784374237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,8191,0.025573333104451496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,16383,0.042362665136655174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,32767,0.09985599915186565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,32767,0.07282133400440216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1,0.012842666357755661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,65535,0.1843679944674174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,65535,0.12642133235931396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,float16,131071,0.3546239932378133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,8,8,128,1,float16,fp8,131071,0.2344533403714498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,1,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,15,0.012784000486135483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,3,0.012661332885424295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,15,0.013440000514189402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,3,0.013477332890033722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,7,0.013290667285521826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,63,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,7,0.01333333303531011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,31,0.012986666212479273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,31,0.013061333447694778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,63,0.013253333667914072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,127,0.01309866706530253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,127,0.01471466695268949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,255,0.014544000228246054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,255,0.015674666812022526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,511,0.0163680004576842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,511,0.018090666582187016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,1023,0.021269333859284718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,4095,0.060085331400235496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,1023,0.022826666633288067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,2047,0.03760000069936117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,2047,0.03295466552178065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,4095,0.05484800040721893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1,0.01310933381319046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,8191,0.1086133321126302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,8191,0.09351999560991923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,float16,16383,0.23986132939656576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,1,128,1,float16,fp8,16383,0.1726613243420919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,3,0.014111999422311783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,1,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,3,0.012981332838535309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,7,0.013007999708255133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,7,0.013818666338920593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,15,0.012981332838535309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,15,0.013674666484196981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,31,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,31,0.014080000420411428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,63,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,63,0.013754667093356451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,127,0.01349866638580958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,127,0.015119999647140503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,255,0.014666666587193808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,255,0.01562133307258288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,511,0.017504000415404636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,511,0.018624000251293182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,2047,0.040896000961462654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,1023,0.034474665919939675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,1023,0.02497066557407379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,2047,0.05709333221117655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,4095,0.09976533055305481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,4095,0.06391466657320659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,8191,0.18370666106541952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,8191,0.11429867148399353
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,float16,16383,0.35174401601155597
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1,0.01691199963291486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,7,0.016976000120242436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,2,128,1,float16,fp8,16383,0.20267732938130698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,3,0.016965333372354507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,1,0.018144000321626663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,3,0.017898666361967724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,7,0.018085333208243053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,63,0.016970666746298473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,15,0.01820266619324684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,15,0.01691199963291486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,31,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,31,0.018207999567190807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,63,0.018101333330074947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,127,0.017680000513792038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,127,0.02090666691462199
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,255,0.021018666525681812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,1023,0.05931733548641205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,255,0.022287999590237934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,1023,0.044394666949907936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,511,0.03737599899371465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,511,0.028357334434986115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,2047,0.10108799735705058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,2047,0.06765866776307423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,4095,0.18412800629933676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,4095,0.11522666613260905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1,0.023941333095232647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,8191,0.3503093322118123
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,8191,0.20675732692082724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,1,0.025621332228183746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,3,0.02367999901374181
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,fp8,16383,0.38622931639353436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,3,0.025258667767047882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,4,128,1,float16,float16,16383,0.6896320184071859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,15,0.025392000873883564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,7,0.023786666492621105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,7,0.025477332373460133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,15,0.023978665471076965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,31,0.025231999655564625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,63,0.024298667907714844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,31,0.02447466552257538
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,63,0.025663999219735462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,127,0.024703999360402424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,127,0.029904000461101532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,255,0.04144533226887385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,511,0.062128002444903054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,255,0.03444266567627589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,511,0.05101866523424784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,1023,0.07320533196131389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,1023,0.10284266869227092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,2047,0.1841119925181071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,2047,0.11910399794578552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,4095,0.3516266743342082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,4095,0.20890132586161295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1,0.01097600037852923
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,1,0.011434666812419891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,8191,0.39020800590515137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,3,0.01081066702802976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,3,0.01137599969903628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,8191,0.6880479653676351
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,7,0.010837333897749582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,fp8,16383,0.7495466868082682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,8,8,128,1,float16,float16,16383,1.4812960624694824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,63,0.01081066702802976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,7,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,63,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,15,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,15,0.011434666812419891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,255,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,31,0.010965333630641302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,31,0.011642667154471079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,127,0.012634667257467905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,127,0.011221333096424738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,255,0.011786667009194693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,511,0.013690666606028875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,511,0.013749333719412485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,1023,0.01470400020480156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,1023,0.014794666320085526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,2047,0.015610666324694952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,2047,0.017557332913080852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,16383,0.05110399921735128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,4095,0.020448000480731327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,4095,0.02189333240191142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,8191,0.028549333413441975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,8191,0.03234133372704188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,65535,0.17150400082270303
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,16383,0.04643733302752177
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,32767,0.08195733527342479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,32767,0.09076266487439473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,65535,0.15013866623242697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,float16,131071,0.2861706614494324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,1,128,1,float16,fp8,131071,0.32837865749994916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,7,0.011690666278203329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1,0.010842667271693548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,1,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,3,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,3,0.011567999919255575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,7,0.011039999624093374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,15,0.010954666882753372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,15,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,31,0.01089599976936976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,31,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,63,0.011077333241701126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,63,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,127,0.011157333850860596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,127,0.012965332716703415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,1023,0.016234666109085083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,255,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,255,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,4095,0.02085866779088974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,511,0.013482666263977686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,511,0.015125333021084467
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,1023,0.015344000111023584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,16383,0.048895999789237976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,2047,0.016442666451136272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,2047,0.018144000321626663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,4095,0.022970666488011677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,8191,0.03107733279466629
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,8191,0.03289599965016047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,16383,0.05410666763782501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,32767,0.08363200227419536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,32767,0.10086933771769206
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,65535,0.15633599956830344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,65535,0.18186666568120322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,float16,131071,0.3012266755104065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,2,128,1,float16,fp8,131071,0.3509013255437215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,1,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,3,0.011402666568756104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,3,0.011802667131026586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,7,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,7,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,15,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,15,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,31,0.011541333049535751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,31,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,255,0.013658666362365087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,63,0.01137599969903628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,63,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,127,0.01137599969903628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,127,0.013967999567588171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,255,0.013130666067202887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,511,0.014533333480358124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,511,0.015322666615247726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,4095,0.023546665906906128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,1023,0.015674666812022526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,1023,0.01591466615597407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,2047,0.01681600014368693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,2047,0.01886933296918869
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,4095,0.02229333420594533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,8191,0.0374293327331543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,8191,0.03431999931732813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,65535,0.20196266969045004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,16383,0.06132799883683523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,16383,0.058517331878344216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,32767,0.1086079974969228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,32767,0.10001599788665771
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,65535,0.18173867464065552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,float16,131071,0.38157331943511963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,4,128,1,float16,fp8,131071,0.35119465986887616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,15,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,1,0.012618667135636011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,3,0.011530666301647821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,3,0.013002666334311167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,63,0.011744000017642975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,7,0.011626667032639185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,7,0.012821332861979803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,127,0.013861333330472311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,15,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,31,0.011829332758982977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,31,0.013663999736309052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,63,0.012725333372751871
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,127,0.011994666109482447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,255,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,255,0.014581333845853806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,511,0.014688000082969666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,511,0.015850666910409927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,1023,0.016106666376193363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,1023,0.01637866720557213
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,2047,0.017418666432301205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,16383,0.09937600294748943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,2047,0.01894933357834816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,4095,0.03611200054486593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,32767,0.18333866198857626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,32767,0.12661866346995035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,4095,0.023743999501069386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,8191,0.057248001297314964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,8191,0.04118400067090988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,16383,0.06446399788061778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,65535,0.35574932893117267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,65535,0.2349440058072408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,float16,131071,0.706554651260376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,1,0.014949332922697067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,8,8,128,1,float16,fp8,131071,0.3813120126724243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,7,0.014869333555301031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,3,0.013936000565687815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,3,0.014757333944241205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,7,0.014149333039919535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,15,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,15,0.015546667079130808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,31,0.01505600040157636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,127,0.014720000326633453
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,31,0.014533333480358124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,63,0.014303999642531076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,63,0.014853333433469137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,127,0.01580799991885821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,255,0.017770666629076004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,255,0.017258666455745697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,511,0.019573333362738293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,511,0.021183999876181286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,1023,0.03681600093841553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,1023,0.024725332856178284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,2047,0.05724266668160757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,2047,0.041759997606277466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,8191,0.1845973332722982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,float16,4095,0.1011893351872762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,4095,0.06689600149790446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,1,128,1,float16,fp8,8191,0.11084266503651936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1,0.01759999990463257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,1,0.018618666877349217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,3,0.018885333091020584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,3,0.017317333569129307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,7,0.01739199956258138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,15,0.01786133274435997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,31,0.01762666677435239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,7,0.01926400015751521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,31,0.01882133384545644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,15,0.018559999763965607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,63,0.01777600000301997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,63,0.018735999862353008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,127,0.021386665602525074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,127,0.0179626668492953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,255,0.022474666436513264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,255,0.022954667607943218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,511,0.04028266668319702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,1023,0.04557333389918009
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,511,0.028666667640209198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,1023,0.060602664947509766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,2047,0.10111467043558757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,4095,0.1860533356666565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,2047,0.07016533116499583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,4095,0.11784533659617107
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1,0.02499733368555705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,float16,8191,0.3526826699574788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,2,128,1,float16,fp8,8191,0.20710400740305582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,1,0.025759999950726826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,3,0.02646933247645696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,7,0.02609066665172577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,3,0.0240639994541804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,15,0.02478933334350586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,31,0.024304000039895374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,15,0.025813333690166473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,31,0.02573866645495097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,7,0.02425066630045573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,127,0.025333332518736523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,63,0.025333332518736523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,63,0.02610666553179423
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,127,0.03075733284155528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,511,0.06402666866779327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,255,0.04269866645336151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,255,0.034847999612490334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,511,0.051967998345692955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,1023,0.10321066776911418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,2047,0.1856693426767985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,1023,0.07446933289368947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,2047,0.12212266524632771
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,4095,0.3528746763865153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,4095,0.214026669661204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,1,0.040250666439533234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,3,0.03982399900754293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,7,0.03736533224582672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,7,0.040287998815377556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,float16,8191,0.6885813077290853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,15,0.039962666730086006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1,0.036703998843828835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,15,0.03755199909210205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,3,0.037402667105197906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,4,128,1,float16,fp8,8191,0.3922826846440633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,31,0.03732266773780187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,31,0.04012266546487808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,63,0.0400693342089653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,127,0.05251200000445048
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,127,0.045754666129748024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,63,0.03811199963092804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,255,0.06710933148860931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,255,0.06253333389759064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,511,0.10791466633478801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,1023,0.1891146699587504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,1023,0.12947199741999307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,511,0.08540800213813782
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,2047,0.3519413471221924
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,2047,0.22233066956202188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,4095,0.6874026457468668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,4095,0.40130666891733807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1,0.019744000087181728
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,fp8,8191,0.7622453371683756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,3,0.019823999454577763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,3,0.02054399996995926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,7,0.019589333484570186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,1,0.020554666717847187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,15,0.0194560003777345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,7,0.02083733429511388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,31,0.019621333728233974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,31,0.02056533346573512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,15,0.020869334538777668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,8,8,128,1,float16,float16,8191,1.3548800150553386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,63,0.019802667200565338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,63,0.02063999945918719
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,127,0.020474666108687718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,255,0.025221332907676697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,127,0.023365333676338196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,255,0.026314665873845417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,511,0.04193066557248434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,1023,0.062309334675470986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,1023,0.04850666721661886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1,0.025770666698614757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,1,0.027589333554108936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,511,0.030602666238943737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,float16,2047,0.10367467006047566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,3,0.02587733417749405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,1,128,1,float16,fp8,2047,0.07186666627724965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,7,0.025850666066010792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,7,0.02754666656255722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,15,0.02593066543340683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,31,0.02608533451954524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,3,0.027952000498771667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,63,0.026122666895389557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,63,0.027664000789324444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,15,0.02769600103298823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,31,0.02733866622050603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,127,0.02738133321205775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,127,0.0322826678554217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,255,0.03762666632731756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,255,0.04571199913819631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,511,0.054799998799959816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,511,0.0670666644970576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1,0.03902933249870936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,1023,0.07740800082683563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,1,0.04196266829967499
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,1023,0.10667199889818828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,3,0.04215466479460398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,3,0.03923200070858002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,7,0.03897066662708918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,float16,2047,0.18918399016062418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,15,0.03958400090535482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,7,0.041375999649365745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,2,128,1,float16,fp8,2047,0.12677866220474243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,31,0.04181333382924398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,15,0.041877334316571556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,31,0.03925333420435587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,63,0.040549332896868386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,127,0.04765866696834564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,127,0.05132266879081726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,255,0.06520533561706543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,2047,0.35647467772165936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,511,0.11106666922569275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,511,0.08919999996821086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,255,0.06821333368619283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1,0.0641599992911021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,63,0.041536000867684685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,2047,0.22751466433207193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,1,0.06862933437029521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,float16,1023,0.1910933256149292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,3,0.06870399912198384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,7,0.06465066472689311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,7,0.06806933383146922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,15,0.0651039977868398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,3,0.06464000046253204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,15,0.06862399975458781
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,4,128,1,float16,fp8,1023,0.13291200002034506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,31,0.06894400219122569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,63,0.07041066884994507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,63,0.0695306658744812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,127,0.07659733295440674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,127,0.09130666653315227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,1023,0.35788265864054364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,31,0.06607999900976817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,255,0.11436800161997478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,255,0.10601600011189778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,2047,0.42692800362904865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,1023,0.23860265811284384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,2047,0.6902453104654948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,fp8,511,0.15276267131169638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1,0.033071999748547874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,8,8,128,1,float16,float16,511,0.19963200887044272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,1,0.03416533271471659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,3,0.03401066611210505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,7,0.03256533294916153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,3,0.03331200033426285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,15,0.0344106654326121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,7,0.034287999073664345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,31,0.033999999364217125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,63,0.032831999162832894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,63,0.034058667719364166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,15,0.03252266595760981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,127,0.03640000025431315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,127,0.03967999915281931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,255,0.04494399825731913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,255,0.052671998739242554
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,31,0.03254399945338567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,511,0.07397333284219106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,511,0.0618399977684021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1,0.04515199859937032
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,float16,1023,0.11607467134793599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,3,0.044954667488733925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,1,128,1,float16,fp8,1023,0.08531733353932698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,3,0.04744533201058706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,1,0.04717866579691569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,7,0.046762665112813316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,15,0.04517866671085358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,7,0.04480533301830292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,63,0.04872000217437744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,31,0.045007998744646706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,63,0.04740799963474274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,15,0.04713066418965658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,31,0.04760533571243286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,127,0.05769599974155426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,127,0.058229332168896995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,255,0.07699733475844066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,255,0.07382399837176006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,511,0.12035733461380005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,511,0.09694400429725647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1,0.07133333384990692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,1,0.07562666634718578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,3,0.07580799857775371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,3,0.07146133482456207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,15,0.07562133173147838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,15,0.07150933146476746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,31,0.07428800066312154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,7,0.07106133302052815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,7,0.07528533538182576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,fp8,1023,0.14332266648610434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,31,0.0761706680059433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,2,128,1,float16,float16,1023,0.19985065857569376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,63,0.07992533346017201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,63,0.0784746656815211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,127,0.08569066723187764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,127,0.09922666351000468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,255,0.12143466869990031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,255,0.11752532919247945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,511,0.16665599743525186
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,511,0.20706133047739664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1,0.12285866340001424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,1,0.12939733266830444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,3,0.1223360002040863
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,15,0.12562132875124613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,15,0.12954666217168173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,3,0.1292800009250641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,31,0.12994133432706198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,31,0.1322826643784841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,7,0.1226026713848114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,63,0.1322719951470693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,float16,1023,0.36713067690531415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,63,0.13692800203959146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,4,128,1,float16,fp8,1023,0.25120532512664795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,127,0.16955200831095377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,511,0.3816800117492676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,127,0.14058666427930197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,7,0.1283146639664968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,3,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,3,0.01181866725285848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,7,0.010938666760921478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,7,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,15,0.01089599976936976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,1,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,15,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,31,0.011077333241701126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,31,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,63,0.011034666250149408
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,63,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,127,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,127,0.013114667187134424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,255,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,255,0.21025067567825317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,255,0.013770667215188345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,511,0.014106666048367819
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,1023,0.46189332008361816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1023,0.01623999948302905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,255,0.19925866524378458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,1023,0.01748266691962878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,2047,0.020154666155576706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,2047,0.021914665897687275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,4095,0.03186666717131933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,8191,0.04669333497683207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,8191,0.051216001311937966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,16383,0.08158933122952779
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,16383,0.09211732943852742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,32767,0.14948800206184387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,32767,0.17035732666651407
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,65535,0.2887519995371501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,1,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,65535,0.32478400071461994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,131071,0.5597600142161051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,fp8,511,0.2901386618614197
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,511,0.014597332725922266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,fp8,131071,0.6399466594060262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,1,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,3,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,7,0.011663999408483505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,15,0.011546666423479715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,7,0.012256000190973282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,15,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,31,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,31,0.01138666644692421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,63,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,63,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,1,128,1,float16,float16,4095,0.02855466554562251
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,127,0.01138666644692421
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,127,0.01368533323208491
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,255,0.012485332787036896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,255,0.015509333461523056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,511,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,511,0.016143999993801117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,8,8,128,1,float16,float16,1023,0.7004266579945883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,1023,0.01613866661985715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,1023,0.018101333330074947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,2047,0.020975999534130096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,2047,0.02292799949645996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,8191,0.05442666510740916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,3,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,4095,0.03073599934577942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,8191,0.048394665122032166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,4095,0.03246400008598963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,16383,0.08372799555460612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,65535,0.29267199834187824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,16383,0.10081066687901814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,65535,0.34913067022959393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,32767,0.15365333358446756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,32767,0.18371200561523438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,float16,131071,0.5696053504943848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,2,128,1,float16,fp8,131071,0.6797173023223877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,3,0.01303999995191892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,1,0.013023999830087027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,7,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,7,0.012714666624863943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,15,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,3,0.01198400060335795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,15,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,31,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,31,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,63,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,511,0.015253332753976187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,63,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,127,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,127,0.0141546664138635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,255,0.014064000298579534
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,255,0.015594666202863058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,1023,0.017125333348910015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,1023,0.01882133384545644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,2047,0.023749334116776783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,2047,0.022250667214393616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,8191,0.06169066826502482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,511,0.016480000068744022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,4095,0.037263999382654824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,4095,0.03573866685231527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,16383,0.09967466195424397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,8191,0.05818133552869161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,32767,0.2018186648686727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,32767,0.1818186640739441
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1,0.012202666451533636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,16383,0.10890666643778484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,65535,0.3895253340403239
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,65535,0.34964799880981445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,1,0.012682666381200155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,3,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,3,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,7,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,fp8,131071,0.6803413232167562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,4,128,1,float16,float16,131071,0.7014346917470297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,7,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,15,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,15,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,31,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,255,0.013568000247081121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,31,0.012773333738247553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,63,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,63,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,127,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,127,0.014058666924635569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,511,0.01621333385507266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,1023,0.019029332945744198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,255,0.015642666568358738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,1023,0.018954666952292126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,2047,0.02638400097688039
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,511,0.01523200049996376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,2047,0.036602665980656944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,4095,0.04058666775623957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,4095,0.05670933425426483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,8191,0.09990400075912476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,32767,0.3550453186035156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,8191,0.06348800162474315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,16383,0.1841119925181071
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,16383,0.10971732934315999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,65535,0.38097067674001056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,32767,0.20042665799458823
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,65535,0.7051253318786621
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,fp8,131071,0.7458613713582357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,8,8,128,1,float16,float16,131071,1.4949493408203125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,1,0.03636800001064936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,1,0.0372533326347669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,7,0.03736533224582672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,7,0.03669866671164831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,15,0.03643200049797694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,15,0.03681600093841553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,3,0.0365226666132609
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,3,0.03717333326737086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,31,0.036373332142829895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,31,0.03698666642109553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,63,0.038704000413417816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,63,0.0372533326347669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,127,0.045226668318112694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,127,0.04739200075467428
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,511,0.10711466272672017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,1,0.061850666999816895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,float16,255,0.06437866886456807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,511,0.08452799916267395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,1,128,1,float16,fp8,255,0.06262399752934773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,3,0.06390400230884552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,3,0.06196799874305725
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,7,0.06328533093134563
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,1,0.06342933575312297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,7,0.06205866734186808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,31,0.06871999800205231
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,15,0.06198933223883311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,15,0.06390933195749919
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,31,0.062463998794555664
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,63,0.07065600156784058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,63,0.06620266536871593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,127,0.07593599955240886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,127,0.08621866504351298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,255,0.1116480032602946
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,255,0.10532266894976298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,1,0.11359999577204387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,1,0.11178132891654968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,fp8,511,0.15365866820017496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,2,128,1,float16,float16,511,0.19574934244155884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,3,0.11455999811490376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,3,0.11202133695284526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,15,0.12060266733169556
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,15,0.11209066708882649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,7,0.11574400464693706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,7,0.11199999849001567
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,63,0.1265120009581248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,63,0.12285332878430684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,31,0.12474133570988973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,31,0.1185653309027354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,127,0.15611732999483743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,1,0.2101973295211792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,1,0.2047040065129598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,255,0.199562668800354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,255,0.1873706579208374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,127,0.13343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,3,0.20885332425435385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,3,0.20440532763799033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,float16,511,0.37146135171254474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,4,128,1,float16,fp8,511,0.2797173261642456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,7,0.2195733388264974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,7,0.20674665768941244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,31,0.22583999236424765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,15,0.21272534132003784
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,31,0.2198293407758077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,15,0.2265226642290751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,63,0.23008533318837485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,127,0.29106666644414264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,127,0.24383467435836792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,63,0.2206453283627828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,255,0.37029866377512616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,1,0.061887999375661217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,255,0.3395306666692098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,1,0.06302933394908905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,3,0.062277331948280334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,3,0.061994666854540505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,fp8,511,0.5219893455505371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,8,8,128,1,float16,float16,511,0.7236959934234619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,7,0.06141333281993866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,7,0.0629013329744339
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,15,0.0629120022058487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,31,0.06583466629187266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,31,0.06280000011126201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,15,0.06149866680304209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,127,0.07565866907437642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,127,0.0851146678129832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,63,0.07045333087444305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,63,0.0690880020459493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,float16,255,0.11413333813349406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,1,0.11523200074831645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,1,128,1,float16,fp8,255,0.1093386709690094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,1,0.11145599683125813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,3,0.11432533462842305
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,3,0.11219732960065205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,7,0.11520533760388692
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,7,0.11247467001279195
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,15,0.1246613363424937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,15,0.11242666840553284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,31,0.12518399953842163
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,31,0.11934933066368103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,63,0.12661866346995035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,63,0.125082661708196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,127,0.13578133781750998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,1,0.21596266825993857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,127,0.15742933750152588
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,float16,255,0.20422399044036865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,1,0.21070400873819986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,2,128,1,float16,fp8,255,0.19325866301854452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,3,0.21653334299723306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,3,0.21248533328374228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,7,0.23148800929387411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,7,0.21172267198562622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,15,0.23598933219909668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,31,0.2343733310699463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,63,0.23720532655715942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,31,0.22978667418162027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,63,0.2300693392753601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,15,0.22849599520365396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,127,0.2536533276240031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,127,0.2958933313687642
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,float16,255,0.3821973403294881
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,1,0.39680532614390057
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,3,0.4291413227717082
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,3,0.4012053410212199
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,4,128,1,float16,fp8,255,0.3554399808247884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,7,0.4394933382670085
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,1,0.41044267018636066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,15,0.42393068472544354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,31,0.43829866250356037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,31,0.4277973175048828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,63,0.42551998297373456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,7,0.4144373337427775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,15,0.44097065925598145
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,1,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,3,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,3,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,7,0.011114666859308878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,7,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,15,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,63,0.443178653717041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,15,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,31,0.011168000598748526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,31,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,63,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,127,0.4708319902420044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,127,0.012837332983811697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,127,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,255,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,255,0.013760000467300415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,511,0.017722666263580322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,1023,0.018719999740521114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,1023,0.021151999632517498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,2047,0.030933332939942677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,4095,0.04571733375390371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,2047,0.02752533306678136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,4095,0.050213331977526345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,8191,0.09119466940561931
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,8191,0.08122666676839192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,16383,0.1486026644706726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,16383,0.1701386570930481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,32767,0.28886399666468304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,32767,0.32435200611750287
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,63,0.011434666812419891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,65535,0.5624906619389852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,fp8,65535,0.6403359969456991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1,0.011711999773979187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,1,0.012608000387748083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,3,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,3,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,7,0.011600000162919363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,7,0.012495999534924826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,15,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,15,0.012618667135636011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,31,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,31,0.012495999534924826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,63,0.011754666765530905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,63,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,127,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,127,0.013946666071812311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,255,0.013546666751305262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,255,0.015168000012636185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,511,0.015450666348139444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,1023,0.019909333437681198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,1,128,1,float16,float16,511,0.01481066644191742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,511,0.017157333592573803
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,1023,0.02189333240191142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,2047,0.028325334191322327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,2047,0.031888000667095184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,4095,0.0496319979429245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,4095,0.052853330969810486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,8191,0.08473066488901775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,8191,0.09852799773216248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,16383,0.1532266636689504
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,127,0.5611466566721598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,16383,0.18289599816004434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,32767,0.29053332408269245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,32767,0.34942400455474854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,float16,65535,0.5748213529586792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,2,128,1,float16,fp8,65535,0.6822186311086019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,1,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,3,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,3,0.012826666235923767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,float16,255,0.7284746964772543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,8,8,128,1,float16,fp8,255,0.6713813145955404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,7,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,7,0.012831999609867731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,15,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,15,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,31,0.01166933278242747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,63,0.013077333569526672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,127,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,511,0.015487999965747198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,127,0.0144213338692983
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,255,0.014490666488806406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,255,0.014959999670584997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,511,0.017530667285124462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,1023,0.020026666422684986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,1023,0.022448000808556873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,2047,0.041146665811538696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,2047,0.033610666791598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,4095,0.059402664502461754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,4095,0.05690133571624756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,8191,0.10098133484522502
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,8191,0.09844266374905904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,16383,0.20012799898783365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,16383,0.18128534158070883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,32767,0.387664000193278
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,32767,0.3492639859517415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,65535,0.7597333590189616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,1,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,3,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,31,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,fp8,65535,0.6795039971669515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,3,0.01321600005030632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,7,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,7,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,31,0.01332266628742218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,15,0.012602667013804117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,15,0.013546666751305262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,127,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,4,128,1,float16,float16,63,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,31,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,63,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,63,0.013445333888133367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,127,0.014896000425020853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,255,0.014378666877746582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,255,0.015562667200962702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,511,0.017114666601022083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,511,0.017925333231687546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,1023,0.035173334181308746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,4095,0.06185600161552429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,1023,0.02348266790310542
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,2047,0.05593066910902659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,2047,0.03982399900754293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,4095,0.09865599870681763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,8191,0.18307199080785116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,8191,0.10878400007883708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,16383,0.35363733768463135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,16383,0.20010133584340414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,32767,0.6912960211435953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,32767,0.381279985109965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1,0.011626667032639185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,1,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,fp8,65535,0.7426613171895345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,3,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,3,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,8,8,128,1,float16,float16,65535,1.484938621520996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,7,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,7,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,15,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,15,0.012608000387748083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,31,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,31,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,63,0.011866666376590729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,63,0.012730666746695837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,127,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,127,0.014117332796255747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,255,0.013455999394257864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,255,0.01523200049996376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,511,0.01570133368174235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,511,0.017024000485738117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,1023,0.019941333681344986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,1023,0.021840001145998638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,2047,0.02924799919128418
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,2047,0.03152533372243246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,4095,0.04938133557637533
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,4095,0.0528053343296051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,8191,0.0851200024286906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,8191,0.09874666730562846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,32767,0.348474661509196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,16383,0.1529813309510549
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,16383,0.18262932697931925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,32767,0.292197326819102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1,0.01232533281048139
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,1,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,float16,65535,0.567301352818807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,3,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,3,0.012906666845083237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,1,128,1,float16,fp8,65535,0.6895840167999268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,7,0.012181332955757776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,7,0.012815999488035837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,15,0.012213333199421564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,15,0.012703999876976013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,31,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,31,0.012928000340859095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,63,0.012175999581813812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,255,0.014922666052977243
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,63,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,511,0.017290666699409485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,127,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,1023,0.02197333425283432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,127,0.0141546664138635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,2047,0.032357332607110344
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,255,0.0138026662170887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,511,0.015728000551462173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,1023,0.020997333029905956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,2047,0.0413973331451416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,4095,0.06047999858856201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,16383,0.18683199087778726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,16383,0.18305067221323648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,4095,0.05673066775004069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,8191,0.10566932956377666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,8191,0.09909333785374959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,32767,0.3561066786448161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,32767,0.3545813163121541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,1,0.013365333278973898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,3,0.013242666920026144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,3,0.012346666306257248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,float16,65535,0.6966933409372965
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,2,128,1,float16,fp8,65535,0.6829333305358887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,7,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,7,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,15,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,127,0.012543999900420507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,15,0.013162666310866674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,31,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,63,0.01259200026591619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,31,0.013301332791646322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,63,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,127,0.014885333677132925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,255,0.014165333161751429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,255,0.015397333850463232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,511,0.016901332885026932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,511,0.017845333864291508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,1023,0.0351200004418691
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,4095,0.0628053347269694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,1023,0.023391999304294586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,2047,0.05563200016816457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,2047,0.04002666721741358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,4095,0.0983840028444926
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,8191,0.18265599012374878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,8191,0.11041067043940227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,16383,0.35207466284434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,16383,0.20037333170572916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1,0.011546666423479715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,32767,0.3816959857940674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,32767,0.6900853315989176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,7,0.01098666712641716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,1,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,7,0.012527999778588613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,3,0.010922666639089584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,3,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,fp8,65535,0.7395146687825521
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,4,4,128,1,float16,float16,65535,1.4823999404907227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,63,0.01190399999419848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,15,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,15,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,31,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,31,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,63,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,511,0.014298666268587112
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,1023,0.013594667116800943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,127,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,255,0.012896000097195307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,127,0.014511999984582266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,4095,0.015487999965747198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,255,0.013989333063364029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,8191,0.017877332866191864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,8191,0.019248000035683315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,511,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,1023,0.014293332894643148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,2047,0.014618666221698126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,2047,0.01578666642308235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,4095,0.016293333222468693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,16383,0.022384000321229298
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,16383,0.024517332514127094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,32767,0.031871999303499855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,32767,0.034634667138258614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,1,0.01240533341964086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,65535,0.050106664498647056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,65535,0.05548266569773356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,float16,131071,0.08916800220807393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,7,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,1,128,1,float16,fp8,131071,0.09970133503278096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1,0.01110400011142095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,3,0.010944000134865442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,3,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,7,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,15,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,15,0.012597333639860153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,31,0.011098666737476984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,255,0.013274667163689932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,31,0.012298667182525
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,63,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,63,0.012485332787036896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,127,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,127,0.014085333794355392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,2047,0.014746667196353277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,255,0.014186666657527288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,511,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,511,0.01441066712141037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,1023,0.013434667140245438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,1023,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,8191,0.019893333315849304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,2047,0.01629866659641266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,4095,0.016293333222468693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,32767,0.0334346666932106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,4095,0.01714133347074191
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,8191,0.018863999595244724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,16383,0.02306666721900304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,16383,0.025226667523384094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,32767,0.035349334279696144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,65535,0.053946668903032936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,65535,0.05740800003210703
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,float16,131071,0.09085866808891296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,2,128,1,float16,fp8,131071,0.10699733098347981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1,0.011007999380429586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,1,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,3,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,3,0.012165332833925882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,7,0.010954666882753372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,7,0.012138667205969492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,15,0.011253333340088526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,15,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,31,0.011018666128317514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,31,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,63,0.011488000551859537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,63,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,127,0.011066666493813196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,1023,0.014597332725922266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,127,0.01423466702302297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,255,0.013359999905029932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,255,0.014842666685581207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,511,0.013418667018413544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,511,0.015306666493415833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,1023,0.015578666081031164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,2047,0.01571200042963028
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,2047,0.01666133354107539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,4095,0.01704000060757001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,4095,0.01754666616519292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,8191,0.01889066646496455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,8191,0.019925333559513092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,16383,0.023775999744733173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,16383,0.025802666942278545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,32767,0.03902933249870936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,32767,0.038202665746212006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,65535,0.06044800082842509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,65535,0.06224533418814341
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,3,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,7,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,float16,131071,0.10748799641927083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,4,4,128,1,float16,fp8,131071,0.10823999842007954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1,0.011002667248249054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,1,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,3,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,7,0.012378666549921036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,15,0.01091733326514562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,15,0.012053333222866058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,127,0.014170666535695394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,31,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,31,0.012234666695197424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,63,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,63,0.012554666648308435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,127,0.011071999867757162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,1023,0.015450666348139444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,255,0.01320533330241839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,255,0.0141546664138635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,511,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,4095,0.019013332823912304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,511,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,8191,0.023498666783173878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,1023,0.014293332894643148
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,2047,0.01584533353646596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,2047,0.016421332955360413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,4095,0.017914666483799618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,65535,0.08433066805203755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,8191,0.021231998999913532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,16383,0.029711998999118805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,16383,0.033861334125200905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,32767,0.04795733094215393
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,1,0.011952000359694162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,32767,0.05266133447488149
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,65535,0.09362666805585225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,3,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,float16,131071,0.15222932895024618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,1,128,1,float16,fp8,131071,0.18623467286427817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,15,0.012367999802033106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,3,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,7,0.011007999380429586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,63,0.012256000190973282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,127,0.01121066634853681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,127,0.014111999422311783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,7,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,15,0.011002667248249054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,511,0.013493333011865616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,31,0.011109333485364914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,31,0.012495999534924826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,63,0.011130666981140772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,2047,0.016714667280515034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,2047,0.017418666432301205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,255,0.013493333011865616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,255,0.014453332871198654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,4095,0.020224000016848247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,511,0.015061333775520325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,1023,0.014592000593741735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,1023,0.016575999557971954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,4095,0.018485333770513535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,8191,0.022304000953833263
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,8191,0.025765334566434223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,16383,0.03143466760714849
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,16383,0.03437866767247518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,32767,0.050053333242734276
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,32767,0.056703999638557434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,65535,0.08482133348782857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,65535,0.10181333621342976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,float16,131071,0.15964266657829285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,2,128,1,float16,fp8,131071,0.18579200903574625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,7,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,1,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,3,0.01119999960064888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,3,0.012309333930412928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,7,0.01211200033624967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,15,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,15,0.012639999389648438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,31,0.011530666301647821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,31,0.013429333766301474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,63,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,63,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,511,0.015487999965747198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,1023,0.015872000406185787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,1023,0.01647466669480006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,127,0.011354666203260422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,127,0.014533333480358124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,255,0.013477332890033722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,255,0.015354666858911514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,511,0.014592000593741735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,2047,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,2047,0.017616000026464462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,16383,0.036288000643253326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,4095,0.019215999792019527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,4095,0.020432000358899433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,8191,0.02384000023206075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,8191,0.024666666984558105
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,16383,0.038575999438762665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,32767,0.058917333682378135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,32767,0.059248000383377075
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,65535,0.10959999759991963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,65535,0.10126399993896484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,float16,131071,0.24177066485087076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,4,4,128,1,float16,fp8,131071,0.1821813384691874
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,1,0.013503999759753546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,15,0.01292266696691513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,3,0.012506666282812754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,3,0.013338666409254074
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,7,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,63,0.013408000270525614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,7,0.013823999712864557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,15,0.013386666774749756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,31,0.012698666503032049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,31,0.013882666826248169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,63,0.01268799975514412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,127,0.012773333738247553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,127,0.015034666905800501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,1023,0.02327999969323476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,255,0.01471466695268949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,255,0.015610666324694952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,2047,0.03392533212900162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,4095,0.05870933334032694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,511,0.016255999604860943
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,511,0.018090666582187016
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,1023,0.022687998910744984
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,2047,0.03738133360942205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,4095,0.05783466498057047
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,8191,0.10990400115648906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,8191,0.10104533036549886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,16383,0.20177600781122842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,16383,0.18340800205866495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,fp8,32767,0.35310399532318115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,1,128,1,float16,float16,32767,0.3900800148646037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,7,0.013754667093356451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,3,0.013066666821638743
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,1,0.014090667168299357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,3,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,7,0.013141332815090815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,63,0.013183999806642532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,63,0.013983999689420065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,15,0.013221333424250284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,15,0.013647999614477158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,31,0.013738666971524557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,255,0.014885333677132925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,31,0.013194666554530462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,127,0.013562666873137156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,127,0.015168000012636185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,255,0.015840000162522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,511,0.017743999759356182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,511,0.018229333062966663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,1023,0.035573333501815796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,1023,0.02380799998839696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,2047,0.057002668579419456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,2047,0.04079466561476389
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,4095,0.0990613301595052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,8191,0.11277332901954651
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,4095,0.06408533453941345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,8191,0.1834239959716797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,16383,0.3522026538848877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1,0.017008000363906223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,1,0.01855466639002164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,16383,0.20270933707555136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,3,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,3,0.018005333840847015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,7,0.016869333883126576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,fp8,32767,0.3875573476155599
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,7,0.018181333939234417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,15,0.01802666609485944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,31,0.018133333573738735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,2,128,1,float16,float16,32767,0.6909493605295817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,15,0.017152000218629837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,127,0.01746133342385292
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,31,0.017082666357358296
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,63,0.01725333308180173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,63,0.018063999712467194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,127,0.020853333175182343
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,255,0.022085333863894146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,511,0.028688001135985058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,511,0.03772799919048945
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,255,0.02094399929046631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,1023,0.05823466678460439
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,1023,0.0443146675825119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,4095,0.18531733751296997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,2047,0.09961066643397014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,2047,0.0676746666431427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,4095,0.11461866895357768
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,8191,0.35157867272694904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,8191,0.20569066206614176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1,0.010954666882753372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,1,0.014111999422311783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,3,0.01180800050497055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,7,0.010863999525705973
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,16383,0.3858293294906616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,3,0.010746666540702185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,7,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,16383,0.6885440349578857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,fp8,32767,0.7436800003051758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,15,0.01101333275437355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,4,4,128,1,float16,float16,32767,1.4782293637593586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,15,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,31,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,31,0.011488000551859537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,255,0.013973332941532135
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,63,0.010911999891201654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,63,0.011887999872366587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,127,0.010677333921194077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,127,0.01404800017674764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,255,0.013045333325862885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,511,0.014650666465361914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,511,0.014896000425020853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,1023,0.015642666568358738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,8191,0.02939733366171519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,1023,0.01589866727590561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,2047,0.016858667135238647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,16383,0.05246399839719137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,2047,0.01876266673207283
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,4095,0.021136000752449036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,4095,0.02332799881696701
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,8191,0.03283733377854029
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,16383,0.04720533390839895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,131071,0.29151467482248944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,32767,0.08387733499209087
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,32767,0.09327466289202373
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,float16,65535,0.15319466590881348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,65535,0.18354666233062744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,1,128,1,float16,fp8,131071,0.353434681892395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1,0.010821333775917688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,1,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,3,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,31,0.01101333275437355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,3,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,31,0.011600000162919363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,7,0.011173332730929056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,7,0.011776000261306763
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,15,0.010954666882753372
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,15,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,63,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,63,0.011493333925803503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,127,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,1023,0.016730666160583496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,1023,0.01658133293191592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,127,0.014010666559139887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,2047,0.0195573332409064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,255,0.01351999988158544
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,255,0.014762666076421738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,511,0.014389333625634512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,511,0.016048000504573185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,2047,0.01739199956258138
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,4095,0.021722666919231415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,4095,0.023984000086784363
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,8191,0.031008000175158184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,8191,0.034474665919939675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,16383,0.04985600213209788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,16383,0.05464000006516775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,32767,0.08692266543706258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,32767,0.10110933581988017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,65535,0.15780799587567648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,1,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,65535,0.1848106582959493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,float16,131071,0.29626667499542236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,2,128,1,float16,fp8,131071,0.35150933265686035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1,0.011173332730929056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,3,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,3,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,7,0.011503999431928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,7,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,15,0.01139733319481214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,15,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,31,0.011173332730929056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,31,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,63,0.011359999577204386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,63,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,127,0.011253333340088526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,127,0.015050667027632395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,255,0.01413333291808764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,255,0.014858666807413101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,511,0.016058667252461117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,511,0.01666133354107539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,4095,0.02446399877468745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,1023,0.01642666632930438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,1023,0.017231999586025875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,2047,0.018298666924238205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,2047,0.019813333948453266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,4095,0.02248000105222066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,8191,0.038922667503356934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,8191,0.03638399889071783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,16383,0.07213333249092102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,65535,0.18392000595728555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,16383,0.05876799921194712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,131071,0.35972265402475995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,32767,0.10957866907119751
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,32767,0.10117333134015401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,float16,65535,0.1970400015513102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,4,4,128,1,float16,fp8,131071,0.3482133150100708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1,0.01370666672786077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,1,0.01544533297419548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,3,0.014997333288192749
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,3,0.014335999886194864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,7,0.014208000153303146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,7,0.015098666151364645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,15,0.013909333695967993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,31,0.015295999745527903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,15,0.014991999914248785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,31,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,63,0.013983999689420065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,127,0.014544000228246054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,63,0.015493333339691162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,127,0.016623999923467636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,255,0.016234666109085083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,255,0.017237332959969837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,511,0.018186666071414948
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,2047,0.05841066439946493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,511,0.019930666933457058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,1023,0.03679466744263967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,1023,0.025920001169045765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,2047,0.04241600135962168
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,4095,0.10084799925486247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,8191,0.18478933970133463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,4095,0.06697600086530049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,8191,0.11365333199501038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,float16,16383,0.3536213239034017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1,0.017711999515692394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,1,128,1,float16,fp8,16383,0.2053813338279724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,3,0.018496000518401463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,1,0.018826667219400406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,3,0.017829333742459614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,15,0.017562666287024815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,15,0.01883200059334437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,7,0.018570666511853535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,31,0.017893332988023758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,31,0.01882133384545644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,7,0.017525333911180496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,63,0.01802666609485944
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,63,0.019023999571800232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,127,0.017973333597183228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,127,0.02189333240191142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,255,0.021669333179791767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,255,0.022778667509555817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,511,0.03893866638342539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,511,0.028864001234372456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,1023,0.060736000537872314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,1023,0.04614399870236715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,2047,0.10085866848627727
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,4095,0.18453866243362427
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,2047,0.07044800122578938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,4095,0.11711999773979187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,8191,0.3529599905014038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,8191,0.20680532852808634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,1,0.02568000058333079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,3,0.025733334322770435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,7,0.024218666056791942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1,0.024101334313551586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,7,0.025744001070658367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,15,0.02422933280467987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,fp8,16383,0.388592004776001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,15,0.02568000058333079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,2,128,1,float16,float16,16383,0.6900320053100586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,3,0.024746666351954143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,31,0.024362665911515553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,31,0.025754667818546295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,63,0.024400000770886738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,127,0.025834667185942333
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,63,0.02608533451954524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,255,0.04270400106906891
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,255,0.03713600089152654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,127,0.0308746670683225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,511,0.06293333570162456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,511,0.051407997806866966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,2047,0.1862773299217224
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,1023,0.10337066650390625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,1023,0.0743093341588974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,2047,0.12154666582743327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,4095,0.3511679967244466
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,4095,0.21185600757598877
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,1,0.020666666328907013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,3,0.01960533360640208
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,8191,0.6874933242797852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,8191,0.3922026554743449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,float16,16383,1.4695520401000977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,3,0.020725333442290623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1,0.019658666104078293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,7,0.019365333020687103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,7,0.02091199904680252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,4,4,128,1,float16,fp8,16383,0.7513973712921143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,15,0.019343999524911244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,15,0.020821332931518555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,31,0.0207893339296182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,63,0.01931200052301089
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,63,0.02091199904680252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,31,0.02019199977318446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,127,0.020245333512624104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,255,0.024319998919963837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,255,0.02526933451493581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,511,0.04258666435877482
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,511,0.03156266609827677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,127,0.023242667317390442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,1023,0.06294933458169301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,1,0.027258666853109997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,1023,0.04923200110594431
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1,0.025983999172846477
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,3,0.025642665723959606
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,2047,0.07353599866231282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,2047,0.10454932848612468
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,3,0.027301333844661713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,7,0.02757333219051361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,15,0.02587733417749405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,31,0.02736533433198929
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,7,0.026047999660174053
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,float16,4095,0.1880693236986796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,1,128,1,float16,fp8,4095,0.11905599633852641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,63,0.02606933315594991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,63,0.0276053324341774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,31,0.026144000391165417
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,15,0.027514666318893433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,127,0.028207999964555103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,127,0.0322080006202062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,511,0.06549866497516632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,511,0.05402133365472158
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,4095,0.35485867659250897
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,255,0.045125335454940796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,1023,0.10610666871070862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,4095,0.21410665909449259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1,0.03870933254559835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,1023,0.07713066538174947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,255,0.036544000109036766
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,1,0.04131199916203817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,float16,2047,0.18786666790644327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,3,0.04153066625197729
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,2,128,1,float16,fp8,2047,0.12585600217183432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,7,0.03874133278926214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,3,0.038986665507157646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,15,0.03882133215665817
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,7,0.041573333243529
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,15,0.04138133426507314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,31,0.041850666205088295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,63,0.0420959989229838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,127,0.04855999847253164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,127,0.05083733300367991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,31,0.03908266623814901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,255,0.0678719977537791
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,255,0.06474666794141133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,2047,0.35496532917022705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,511,0.11001599828402202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,511,0.0888213316599528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,63,0.04058133314053217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,2047,0.22712532679239908
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,4095,0.4049599965413411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,4095,0.6885973612467448
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,float16,1023,0.18986133734385172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,4,4,128,1,float16,fp8,1023,0.13170133034388223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1,0.03109866629044215
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,1,0.03257066756486893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,3,0.031152000029881794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,3,0.0327360009153684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,7,0.031093334158261616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,7,0.032613334556420646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,15,0.031194667021433514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,15,0.032842665910720825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,31,0.03127466638882955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,31,0.03305066625277201
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,63,0.031136001149813335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,63,0.03268799930810928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,127,0.03252266595760981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,127,0.036981334288915
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,255,0.049327999353408813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,255,0.04295999805132548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,511,0.06054399907588959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,511,0.07249066730340321
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,1023,0.11265599727630615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,1,0.045456002155939736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1,0.04261333247025808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,1023,0.08321600159009297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,3,0.04251733422279358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,3,0.045925334095954895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,7,0.04283200204372406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,7,0.04550399879614512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,float16,2047,0.1941173275311788
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,15,0.042693331837654114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,1,128,1,float16,fp8,2047,0.13077333569526672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,15,0.044906665881474815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,31,0.04327466587225596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,31,0.04562133550643921
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,63,0.0444213350613912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,63,0.045456002155939736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,127,0.05470400055249532
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,255,0.07046400010585785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,255,0.07683733105659485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,511,0.1164533297220866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,127,0.05518933137257894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,511,0.09583999713261922
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,3,0.0689279983441035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1,0.06934399902820587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,1023,0.13939733306566873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,1,0.07421333094437917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,1023,0.1971199909845988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,3,0.07437333464622498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,7,0.07397333284219106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,7,0.06915199756622314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,15,0.07427733143170674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,31,0.07437866429487865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,fp8,2047,0.23291732867558798
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,63,0.0761599987745285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,63,0.07593599955240886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,2,128,1,float16,float16,2047,0.36052799224853516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,15,0.06919999917348225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,31,0.07004266480604808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,127,0.08192533254623413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,127,0.0976106623808543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,255,0.1143893301486969
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,511,0.1611253321170807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,2047,0.4373013178507487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,3,0.011039999624093374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,2047,0.6970240275065104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,3,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,7,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,15,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,15,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,31,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,31,0.01219733307758967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,511,0.20142932732899985
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,7,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,63,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,63,0.011989332735538483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,127,0.012970666090647379
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,1,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,255,0.012426666915416718
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,255,0.11863999565442403
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,255,0.015237333873907724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,511,0.014554666976133982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,511,0.016373333831628162
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,1023,0.017045332739750545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,2047,0.02124800036350886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,2047,0.0234400009115537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,4095,0.029525332152843475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,4095,0.03390933324893316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,8191,0.04818133513132731
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,127,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,1023,0.018794666975736618
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,8191,0.05250666538874308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,fp8,1023,0.24749332666397095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,4,4,128,1,float16,float16,1023,0.362005352973938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,16383,0.0832586685816447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,16383,0.09684266646703084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,32767,0.15365333358446756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1,0.011242666592200598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,32767,0.18477332592010498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,65535,0.28893866141637164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,65535,0.35285866260528564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,1,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,3,0.011525332927703857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,3,0.012133333832025528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,7,0.011493333925803503
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,7,0.011930666863918304
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,15,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,15,0.011946666985750198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,31,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,31,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,63,0.011391999820868174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,63,0.012128000458081564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,float16,131071,0.5636906623840332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,127,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,127,0.013429333766301474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,255,0.012602667013804117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,255,0.015935999651749928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,511,0.015402667224407196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,2047,0.022042666872342426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,1,128,1,float16,fp8,131071,0.6881600220998129
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,4095,0.03092266619205475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,511,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,4095,0.03378133227427801
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,1023,0.01764800027012825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,1023,0.019109333554903667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,16383,0.10108266274134318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,2047,0.023919999599456787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,8191,0.050245334704717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,65535,0.29148266712824505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,8191,0.055173332492510475
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,16383,0.08501866459846497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,32767,0.1551093359788259
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,32767,0.18572266896565756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,65535,0.34994665781656903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,float16,131071,0.5697439908981323
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,7,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1,0.011648000528415045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,2,128,1,float16,fp8,131071,0.6830986340840658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,1,0.012560000022252401
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,3,0.011509332805871964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,3,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,7,0.012661332885424295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,63,0.012527999778588613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,15,0.011717333147923151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,15,0.012800000607967377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,31,0.011674666156371435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,31,0.01293333371480306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,63,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,511,0.017210666090250015
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,127,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,127,0.0141546664138635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,255,0.013210666676362356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,255,0.016506666938463848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,511,0.016271999726692837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,1023,0.017759999881188076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,8191,0.07276799778143565
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,8191,0.0589279979467392
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,1023,0.019333332777023315
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,16383,0.10052800178527832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,2047,0.02205866575241089
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,2047,0.02420266717672348
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,4095,0.04329599936803182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,4095,0.03562133262554804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,16383,0.12864533066749573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,32767,0.20301334063212076
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,32767,0.18359466393788657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,65535,0.46776000658671063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,65535,0.34730132420857746
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,float16,131071,0.7098293304443359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,4,4,128,1,float16,fp8,131071,0.6777973175048828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1,0.03732266773780187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,1,0.03818666686614355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,3,0.038032000263532005
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,7,0.03721066564321518
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,7,0.03817066550254822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,15,0.03738666574160258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,15,0.03782933453718821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,31,0.03737599899371465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,31,0.03797333439191183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,3,0.03759466608365377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,63,0.039274667700131737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,63,0.03806933263937632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,127,0.04535999894142151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,127,0.04684799909591675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,255,0.06410666803518932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,255,0.0633546660343806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,511,0.10583999752998352
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1,0.06322666505972545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,511,0.08717333277066548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,1,0.06247466802597046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,3,0.06342400113741557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,3,0.0618399977684021
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,float16,1023,0.1888479987780253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,1,128,1,float16,fp8,1023,0.1302880048751831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,7,0.06189866860707601
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,7,0.06359466910362244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,15,0.0626453310251236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,15,0.06371200084686279
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,31,0.06617600222428639
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,31,0.06208533545335134
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,127,0.07433600227038066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,127,0.08622399965922038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,63,0.07077866792678833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,255,0.10932266712188721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,63,0.06600533425807953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,255,0.10526933272679646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,511,0.1927679975827535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1,0.11518399914105733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,3,0.11534399787584941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,511,0.15238933761914572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,1,0.11161067088445027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,3,0.11318932970364888
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,float16,1023,0.35629332065582275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,2,128,1,float16,fp8,1023,0.24007999897003174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,7,0.1132426659266154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,7,0.11406933267911275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,15,0.11983467141787212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,31,0.12381333112716675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,31,0.11780800422032674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,15,0.11231467127799988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,127,0.13310399651527405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,127,0.1560373306274414
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,255,0.19806400934855142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,63,0.12304000059763591
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,255,0.18513067563374838
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,63,0.12522133191426596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,511,0.37057065963745117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,511,0.2820906639099121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,1,0.06515199939409892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,3,0.06496533254782359
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,1,0.06483733157316844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,3,0.06366399923960368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,float16,1023,0.7681120236714681
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,4,4,128,1,float16,fp8,1023,0.44471999009450275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,15,0.06452799836794536
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,15,0.06515199939409892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,7,0.0644053320089976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,7,0.06485866506894429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,31,0.06498133142789204
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,31,0.06849066913127899
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,127,0.07726400097211202
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,127,0.0879146655400594
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,255,0.11105599999427795
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,255,0.11222933729489644
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,511,0.19486399491628012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,63,0.06965333223342896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,float16,63,0.07177599767843883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,1,128,1,float16,fp8,511,0.1593280037244161
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,1,0.11522666613260905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,1,0.11178666353225708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,3,0.1148533324400584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,3,0.11327466368675232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,7,0.11163199941317241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,15,0.12148267030715942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,7,0.114464004834493
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,31,0.12595733006795248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,31,0.11705066760381062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,63,0.12616533041000366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,63,0.12447466452916463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,15,0.11329066753387451
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,127,0.1339306632677714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,127,0.15659733613332114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,255,0.19991467396418253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,255,0.19134400288263956
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,1,0.2174773414929708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,float16,511,0.3705600102742513
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,1,0.21346133947372437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,2,128,1,float16,fp8,511,0.2871039907137553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,3,0.2156213323275248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,3,0.21198934316635132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,7,0.22444800535837808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,15,0.23598933219909668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,7,0.21252266565958658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,31,0.23720532655715942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,63,0.2359573245048523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,63,0.2304853399594625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,15,0.22060799598693848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1,0.012602667013804117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,1,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,3,0.011205332974592844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,31,0.22963200012842813
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,3,0.012058666596810022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,7,0.012069333344697952
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,15,0.011343999455372492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,15,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,7,0.011050666371981302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,31,0.01129066695769628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,31,0.012005332857370377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,63,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,63,0.012042666474978128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,127,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,127,0.013199999928474426
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,127,0.29895466566085815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,255,0.012810666114091873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,511,0.016901332885026932
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,1023,0.018954666952292126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,1023,0.021397332350413006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,2047,0.027461332579453785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,127,0.24963732560475668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,2047,0.031130666534105938
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,255,0.3574133316675822
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,511,0.015168000012636185
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,255,0.014138666292031607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,4095,0.05035733183224996
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,8191,0.08239999910195668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,8191,0.09328533212343852
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,4095,0.04573333263397217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,16383,0.14998400211334229
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,16383,0.18305067221323648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,32767,0.2889066735903422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,32767,0.3484213352203369
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,255,0.37744001547495526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,65535,0.5620266596476237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,65535,0.6864960193634033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,3,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,float16,131071,1.123685359954834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,1,128,1,float16,fp8,131071,1.338479995727539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,3,0.012618667135636011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,7,0.011823999385039011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,7,0.01312000056107839
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,15,0.011936000237862269
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,15,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,fp8,511,0.5412106513977051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,31,0.011978667229413986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,1,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,31,0.012944000462690989
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,63,0.011754666765530905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,127,0.011877333124478659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,4,4,128,1,float16,float16,511,0.760037342707316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,63,0.012645332763592402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,255,0.013381333400805792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,255,0.014592000593741735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,511,0.01544533297419548
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,511,0.017909333109855652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,2047,0.032442666590213776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,1023,0.01956266661485036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,1023,0.02237333357334137
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,4095,0.048800001541773476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,4095,0.05226133267084757
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,8191,0.08366933465003967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,8191,0.09883733590443929
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,16383,0.15228266517321268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,16383,0.18339200814565024
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,32767,0.29097066322962445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,32767,0.34564801057179767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,65535,0.5776373147964478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,65535,0.6813333034515381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,1,0.01310933381319046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,3,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,3,0.013013333082199097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,131071,1.1200319925944011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,7,0.011610666910807291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,7,0.012917333592971167
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,131071,1.351568063100179
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,15,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,31,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,15,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,31,0.012960000584522883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,63,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,63,0.012863999853531519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,127,0.012810666114091873
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,127,0.014533333480358124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,255,0.01392000044385592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,255,0.015066667149464289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,511,0.01543466622630755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,511,0.017845333864291508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,2047,0.035631999373435974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,1023,0.021898667017618816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,4095,0.05946666498978933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,fp8,127,0.01443733274936676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,1023,0.022367998957633972
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,2047,0.03364799916744232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,4095,0.05663999915122986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,8191,0.10565333565076192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,2,128,1,float16,float16,2047,0.029477333029111225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,8191,0.09940266609191895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,16383,0.18577067057291666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,16383,0.18198933204015097
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,32767,0.4621653159459432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,65535,0.7093066374460856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,32767,0.34827733039855957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,65535,0.6794986724853516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,3,0.011962667107582092
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,7,0.011802667131026586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,1,0.01231466606259346
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,float16,131071,1.5049707094828289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,3,0.012703999876976013
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,4,4,128,1,float16,fp8,131071,1.3482666015625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,7,0.01267733300725619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,15,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,15,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,31,0.011584000041087469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,31,0.012437333663304647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,63,0.011792000383138657
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,63,0.012533333152532578
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,127,0.012261333564917246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,127,0.014688000082969666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,1023,0.02004266654451688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,1023,0.021776000658671062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,255,0.014405333747466406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,255,0.014912000546852747
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,511,0.015178666760524115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,511,0.01706133286158244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,2047,0.029322666426499683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,2047,0.03178133318821589
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,4095,0.04977599779764811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,4095,0.052330667773882546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,8191,0.08251200119654338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,8191,0.09891200065612793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,16383,0.15634133418401083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,16383,0.18413333098093668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,32767,0.2945760091145833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,32767,0.3493013381958008
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,65535,0.5670400063196818
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,65535,0.6786613464355469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1,0.012319999436537424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,1,0.012847999731699625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,3,0.012858666479587555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,3,0.01227733368674914
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,fp8,131071,1.348314603169759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,1,128,1,float16,float16,131071,1.1317226886749268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,15,0.012847999731699625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,7,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,7,0.012975999464591345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,15,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,31,0.012085333466529846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,31,0.014314666390419006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,255,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,63,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,127,0.012448000411192576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,63,0.012773333738247553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,127,0.014101333916187286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,255,0.015333333363135656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,511,0.015685333559910457
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,511,0.01728533332546552
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,4095,0.05923733115196228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,1023,0.02085866779088974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,1023,0.02184533327817917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,2047,0.03648533423741659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,2047,0.03347733368476232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,4095,0.056986664732297264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,8191,0.10448533296585083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,32767,0.34721601009368896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,8191,0.09891200065612793
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,16383,0.20006932814915976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,16383,0.18403732776641846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,32767,0.3880426486333211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,65535,0.7608106931050619
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,65535,0.6841333707173666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,1,0.01191466674208641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,3,0.011258666714032492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,3,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,7,0.011061333119869232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,31,0.011370666325092316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,fp8,131071,1.3381439844767253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,2,2,128,1,float16,float16,131071,1.500773270924886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,7,0.012282667060693106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,15,0.0120319997270902
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,127,0.011077333241701126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,15,0.011114666859308878
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,127,0.014394666999578476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,31,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,63,0.011007999380429586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,63,0.012293333808581034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,255,0.013306666165590286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,2047,0.014602666099866232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,255,0.01434133326013883
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,511,0.013242666920026144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,511,0.014549333602190018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,1023,0.013621332744757334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,1023,0.014378666877746582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,16383,0.02250666668017705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,2047,0.0162773331006368
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,4095,0.01573866605758667
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,4095,0.01647466669480006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,8191,0.017978666971127193
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,8191,0.019317333896954853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,16383,0.02491733431816101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,32767,0.032127998769283295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,32767,0.03477333237727483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,65535,0.05105599761009216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,65535,0.055642664432525635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,3,0.012442667037248611
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,float16,131071,0.08931733171145122
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,1,128,1,float16,fp8,131071,0.10032000144322713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,15,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,31,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,1,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,3,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,7,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,127,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,7,0.012389333297808966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,15,0.011055999745925268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,31,0.012400000045696894
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,511,0.01332266628742218
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,63,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,63,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,1023,0.015285332997639975
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,2047,0.015013333410024643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,127,0.01451733335852623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,255,0.013834666460752487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,255,0.014549333602190018
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,511,0.014831999937693277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,8191,0.019088000059127808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,1023,0.013781332721312841
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,2047,0.01632533346613248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,4095,0.016623999923467636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,4095,0.017407999684413273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,8191,0.02000533292690913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,16383,0.02313599983851115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,16383,0.025888000925381977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,32767,0.03287466615438461
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,32767,0.035455999275048576
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,65535,0.05481599768002828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,65535,0.05874133110046387
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,float16,131071,0.092031995455424
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,2,2,128,1,float16,fp8,131071,0.1072106659412384
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1,0.011285333583752314
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,1,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,15,0.012517333030700684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,3,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,3,0.012080000092585882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,7,0.011002667248249054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,7,0.01173866664369901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,15,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,31,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,31,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,63,0.01090666651725769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,63,0.012453333785136541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,127,0.011120000233252844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,127,0.014378666877746582
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,1023,0.015615999698638916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,255,0.013343999783198038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,255,0.014565333724021912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,4095,0.018191999445358913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,511,0.013967999567588171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,8191,0.02143466720978419
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,511,0.014671999961137772
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,1023,0.01492799942692121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,2047,0.01600533351302147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,32767,0.04738666613896688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,2047,0.016672000288963318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,4095,0.019365333020687103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,8191,0.023872000475724537
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,16383,0.03047466774781545
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,16383,0.03390933324893316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,32767,0.052986666560173035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,1,0.01246400053302447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,3,0.011445333560307821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,65535,0.08428800106048584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,65535,0.09427733222643535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,7,0.012144000579913458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,15,0.011157333850860596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,15,0.012362666428089142
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,float16,131071,0.15213867028554282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,1,128,1,float16,fp8,131071,0.1842613418896993
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,3,0.01202133297920227
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,7,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,31,0.011333333949247995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,31,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,63,0.011002667248249054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,63,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,127,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,127,0.014688000082969666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,255,0.013728000223636627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,2047,0.016895999511082966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,255,0.015184000134468079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,4095,0.018687999496857326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,511,0.013967999567588171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,511,0.015594666202863058
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,1023,0.015114666273196539
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,1023,0.016602666427691776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,2047,0.017711999515692394
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,4095,0.019909333437681198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,8191,0.022458667556444805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,8191,0.02478933334350586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,16383,0.032042667269706726
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,16383,0.034741332133611046
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,32767,0.05009066561857859
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,32767,0.0557226687669754
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,65535,0.08521599570910136
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,65535,0.10170666376749675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,float16,131071,0.15585600336392721
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,2,2,128,1,float16,fp8,131071,0.18451199928919473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,7,0.012432000289360682
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1,0.013045333325862885
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,1,0.013429333766301474
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,15,0.013562666873137156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,31,0.0124746672809124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,3,0.012629333883523941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,3,0.013493333011865616
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,7,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,127,0.013056000073750814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,15,0.012949333836634954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,31,0.013690666606028875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,63,0.012661332885424295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,63,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,127,0.015050667027632395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,1023,0.021173333128293354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,255,0.014544000228246054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,255,0.015562667200962702
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,511,0.016442666451136272
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,511,0.018325333793958027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,1023,0.022783999641736347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,2047,0.036533333361148834
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,4095,0.057536001006762184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,2047,0.03439466655254364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,4095,0.06234666705131531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,8191,0.12786666552225748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,8191,0.10160000125567119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,16383,0.18754667043685913
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,16383,0.18393067518870035
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,32767,0.3883039951324463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1,0.012986666212479273
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,32767,0.3563946485519409
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,1,0.013877333452304205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,3,0.01301866645614306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,3,0.014090667168299357
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,float16,65535,0.7633333206176758
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,7,0.014010666559139887
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,7,0.013007999708255133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,1,128,1,float16,fp8,65535,0.6890292962392172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,15,0.014661333213249842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,63,0.01392000044385592
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,15,0.01322666679819425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,31,0.013088000317414602
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,31,0.013674666484196981
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,63,0.013125333935022354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,127,0.013141332815090815
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,255,0.015061333775520325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,127,0.01504533365368843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,255,0.016197333733240765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,1023,0.02425066630045573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,511,0.01752000053723653
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,511,0.01850133389234543
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,4095,0.09861866633097331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,1023,0.03540800015131632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,2047,0.056559999783833824
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,2047,0.040448000033696495
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,4095,0.06449066599210103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,16383,0.3528960148493449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,8191,0.18286933501561484
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,8191,0.11322133739789327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,16383,0.20654400189717612
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1,0.011317333827416102
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,1,0.011733333269755045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,3,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,32767,0.6909120082855225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,3,0.01221866657336553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,15,0.011237333218256632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,7,0.011152000476916632
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,7,0.011701333026091257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,15,0.012383999923865
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,31,0.01118933285276095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,31,0.011978667229413986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,63,0.011343999455372492
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,63,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,127,0.011264000087976456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,127,0.014426667243242264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,255,0.014474666366974512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,255,0.013999999811251959
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,511,0.014757333944241205
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,511,0.015669333438078564
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,32767,0.38532265027364093
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,1023,0.01568000018596649
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,1023,0.01611199975013733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,fp8,65535,0.7567306359608968
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,8191,0.029765332738558452
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,2047,0.0169813334941864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,2047,0.01905599981546402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,2,2,128,1,float16,float16,65535,1.475861390431722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,4095,0.020981334149837494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,4095,0.023562667270501454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,8191,0.03346133232116699
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,16383,0.05260799825191498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,16383,0.04837333162625631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,32767,0.08471999565760295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,32767,0.09527466694513957
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,65535,0.1840533415476481
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,65535,0.15250133474667868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,float16,131071,0.28883200883865356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,1,128,1,float16,fp8,131071,0.35226666927337646
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1,0.01101333275437355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,1,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,3,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,3,0.011594666788975397
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,7,0.011589333415031433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,7,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,63,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,15,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,15,0.011898666620254517
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,31,0.010944000134865442
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,31,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,63,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,127,0.011136000355084738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,127,0.014432000617186228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,255,0.013440000514189402
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,255,0.01509333277742068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,511,0.01479999969402949
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,511,0.01624533285697301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,1023,0.016762666404247284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,1023,0.017658667018016178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,2047,0.017429333180189133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,2047,0.019834666202465694
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,4095,0.021925332645575207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,4095,0.024330665667851765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,8191,0.03065599997838338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,8191,0.034304000437259674
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,16383,0.05240533252557119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,16383,0.054757331808408104
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,32767,0.08784000078837077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,131071,0.2938133279482524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,131071,0.3529119888941447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,32767,0.10116799672444661
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,float16,65535,0.1581653356552124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,2,2,128,1,float16,fp8,65535,0.18344000975290933
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1,0.013967999567588171
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,1,0.014794666320085526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,3,0.0145066666106383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,15,0.013957332819700241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,3,0.015216000378131866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,31,0.014042666802803675
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,7,0.01402666668097178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,7,0.015087999403476715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,63,0.014858666807413101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,15,0.01470400020480156
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,31,0.014954666296641031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,63,0.014368000129858652
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,127,0.014256000518798828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,127,0.016362667083740234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,255,0.015578666081031164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,255,0.01718933383623759
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,511,0.01842133328318596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,2047,0.05783999959627787
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,511,0.019648000597953796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,1023,0.03678400069475174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,1023,0.025418666501839954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,2047,0.04268800218900045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,4095,0.09996267159779866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,8191,0.1851093371709188
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,4095,0.06624533236026764
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,8191,0.11485866705576579
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,16383,0.3543573220570882
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,16383,0.20645866791407266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,1,0.018650667121013004
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1,0.017525333911180496
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,float16,32767,0.6923999786376953
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,3,0.018805333723624546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,7,0.01868266612291336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,7,0.017349333812793095
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,15,0.018565333137909572
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,15,0.017743999759356182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,1,128,1,float16,fp8,32767,0.3869440158208211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,3,0.017866666118303936
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,31,0.018933333456516266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,31,0.017797333498795826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,127,0.021551998953024547
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,63,0.017632000148296356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,63,0.018778666853904724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,127,0.017845333864291508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,511,0.039461334546407066
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,255,0.021301334102948506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,255,0.02309866746266683
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,511,0.029178666571776073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,1023,0.060346667965253196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,1023,0.04552533229192098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,2047,0.10110400120417277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,4095,0.184879998366038
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,2047,0.0694400022427241
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,4095,0.11623467008272807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,8191,0.35332266489664715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,1,0.020901332298914593
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,8191,0.20804800589879355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,3,0.019413333386182785
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,16383,0.6871733665466309
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,3,0.020879998803138733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,16383,0.3856000105539958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,7,0.0200853335360686
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1,0.01959466685851415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,7,0.0207893339296182
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,fp8,32767,0.7628586292266846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,15,0.02085866779088974
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,31,0.021013334393501282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,63,0.019317333896954853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,15,0.01945066700379054
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,63,0.02086399992307027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,31,0.019551999866962433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,2,2,128,1,float16,float16,32767,1.3632799784342449
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,255,0.024319998919963837
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,255,0.025093334416548412
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,511,0.041706666350364685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,127,0.019861333072185516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,511,0.032218667368094124
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,1023,0.06326933205127716
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,2047,0.10357866684595744
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,8191,0.3580319881439209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,2047,0.07340799768765767
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,127,0.02369600037733714
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,8191,0.21242666244506836
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,float16,4095,0.18890132506688437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,3,0.02554133286078771
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,1023,0.0499893327554067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,1,128,1,float16,fp8,4095,0.11936533451080322
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1,0.025994665920734406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,3,0.027301333844661713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,7,0.02754666656255722
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,1,0.027162666122118633
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,15,0.02589866767326991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,15,0.027877333263556164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,31,0.02762666592995326
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,63,0.02757333219051361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,63,0.025994665920734406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,127,0.02681066592534383
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,127,0.03215999901294708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,255,0.045237332582473755
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,7,0.026154667139053345
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,255,0.0366239994764328
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,511,0.05428266525268555
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,511,0.06606400012969971
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,31,0.026213333010673523
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,1023,0.07737599809964497
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,1023,0.10540800293286641
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,4095,0.35445864995320636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,4095,0.21491734186808267
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,2047,0.1879840095837911
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,float16,8191,0.6904053688049316
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1,0.030837332208951313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,8191,0.4000693162282308
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,2,2,128,1,float16,fp8,2047,0.1256533364454905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,1,0.03198933353026708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,3,0.0322773332397143
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,3,0.031023999055226643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,15,0.030896000564098358
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,7,0.03239466746648153
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,31,0.032416000962257385
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,63,0.03108799954255422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,63,0.032816000282764435
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,7,0.03102933367093404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,15,0.032602667808532715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,127,0.03196266790231069
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,127,0.03659199923276901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,255,0.04840533435344696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,255,0.04166933397452036
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,31,0.031152000029881794
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,511,0.07180800040562947
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,511,0.06057066718737284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,1023,0.11249066392580669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,1,0.04539200166861216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,1023,0.08329600095748901
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1,0.042693331837654114
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,3,0.04523199796676636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,7,0.045365333557128906
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,2047,0.1308693289756775
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,15,0.04276266694068909
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,2047,0.19452265898386636
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,3,0.043007999658584595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,15,0.04529066880544027
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,7,0.0421973317861557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,31,0.04273599882920583
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,31,0.04541333516438802
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,float16,4095,0.3595626751581828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,63,0.04568000137805939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,1,128,1,float16,fp8,4095,0.2212053338686625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,127,0.05576533575852712
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,127,0.05500266452630361
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,63,0.04372799893220266
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,255,0.07458133498827617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,255,0.07101333141326904
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,511,0.09569600224494934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,1023,0.1946986714998881
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,1023,0.13920000195503235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,4095,0.41338133811950684
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,3,0.012159999459981918
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,511,0.11759466926256816
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,4095,0.6928213437398275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,7,0.012149333953857422
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,31,0.011194666226704916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,7,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,31,0.012170666207869848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,63,0.01129066695769628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,63,0.01209066684047381
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,15,0.011957333733638128
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,127,0.011247999966144562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,127,0.013151999562978745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,255,0.012223999947309494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,15,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,255,0.015568000574906668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,511,0.015360000232855478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,511,0.016389333953460056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,1023,0.01710933322707812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,1023,0.019029332945744198
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,3,0.010874666273593903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,2047,0.023530667026837666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,1,0.011749333391586939
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,2047,0.021770666042963665
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,4095,0.029989334444204967
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,4095,0.03359466542800268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,fp8,2047,0.2339466611544291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,8191,0.04805333415667216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,2,2,128,1,float16,float16,2047,0.3598986864089966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,8191,0.05230933427810669
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,16383,0.08402666449546814
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,16383,0.09245333075523376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,32767,0.1532799998919169
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,65535,0.2918399969736735
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,float16,131071,0.5631786584854126
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,1,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,131071,0.6865333716074625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,3,0.01126933346192042
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,3,0.011855999628702799
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,7,0.011296000331640244
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,7,0.012240000069141388
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,31,0.011498666057984034
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,15,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,15,0.012106666962305704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,31,0.01184533288081487
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,63,0.011418666690587997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,255,0.012543999900420507
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,63,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,32767,0.1839466691017151
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,127,0.011637333780527115
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,127,0.013386666774749756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,1,128,1,float16,fp8,65535,0.3534613450368245
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,255,0.0164533331990242
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,511,0.015978666643301647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,511,0.01693333312869072
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,1023,0.017664000391960144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,1023,0.01958400011062622
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,8191,0.05449600021044413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,2047,0.022442666192849476
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,2047,0.024304000039895374
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,4095,0.03137599925200144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,4095,0.03389333436886469
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,8191,0.050293331344922386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,65535,0.2950933376948039
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,16383,0.08497066299120586
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,16383,0.10073066751162212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,32767,0.15455466508865356
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,32767,0.18658133347829184
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,65535,0.35206401348114014
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,float16,131071,0.5689653158187866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,2,2,128,1,float16,fp8,131071,0.6835786501566569
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1,0.037615999579429626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,1,0.03801066676775614
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,3,0.037205333511034645
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,3,0.03773866593837738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,7,0.03759466608365377
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,7,0.03835200021664301
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,15,0.03714666763941447
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,15,0.037920000652472176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,31,0.037503999968369804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,31,0.0379573330283165
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,63,0.0377813329299291
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,63,0.03811199963092804
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,127,0.045647998650868736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,255,0.06275199850400288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,255,0.06321600079536438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,127,0.0470773329337438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,511,0.1051626702149709
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,511,0.08582400282224019
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1,0.06287466486295064
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,1,0.06228266656398773
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,1023,0.1880213419596354
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,3,0.063509335120519
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,1023,0.12973333398501077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,3,0.06260799864927928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,7,0.06283199787139893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,float16,2047,0.36205867926279706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,7,0.06216000020503998
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,15,0.062234664956728615
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,31,0.06238399942715963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,1,128,1,float16,fp8,2047,0.224400003751119
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,63,0.06898133456707001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,15,0.06305600206057231
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,31,0.06595199803511302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,127,0.07415466507275899
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,127,0.086325337489446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,255,0.1088746686776479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,63,0.06389866769313812
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,255,0.10425066947937012
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,511,0.19197867314020792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,511,0.15129599968592325
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1,0.06483733157316844
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,1,0.06454933186372121
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,1023,0.23754133780797324
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,1023,0.36139198144276935
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,3,0.06364800035953522
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,7,0.06515199939409892
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,3,0.06493866443634033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,fp8,2047,0.42814401785532635
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,7,0.06473066906134288
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,2,2,128,1,float16,float16,2047,0.7069439888000488
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,31,0.06587733328342438
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,31,0.06557866434256236
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,15,0.06489600241184235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,63,0.07130666573842366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,15,0.06411733229955037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,127,0.07559466858704884
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,255,0.11076800028483073
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,255,0.1104159951210022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,511,0.19287999471028647
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,511,0.15778133273124695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,63,0.06819733480612437
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1,0.11385066310564677
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,1,0.11203733086585999
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,127,0.08872000376383464
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,3,0.11404800415039062
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,fp8,1023,0.243231991926829
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,1,128,1,float16,float16,1023,0.3583466609319051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,3,0.11235733826955159
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,7,0.11524800459543864
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,7,0.1113866666952769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,15,0.11784000198046367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,15,0.1125440001487732
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,31,0.12437867124875386
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,63,0.12422399719556172
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,127,0.13375999530156454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,127,0.15497600038846335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,255,0.19776533047358194
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,255,0.19141334295272827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,63,0.1267626682917277
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,31,0.11458133657773335
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,3,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,3,0.011519999553759893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,7,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1,0.011157333850860596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,7,0.012229333321253458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,15,0.011183999478816986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,15,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,31,0.011616000284751257
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,31,0.011893333246310553
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,63,0.011306667079528173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,63,0.012357333054145178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,127,0.01128000020980835
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,1,0.012256000190973282
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,127,0.013386666774749756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,255,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,255,0.014240000396966934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,511,0.016688000410795212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,511,0.36669333775838214
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,511,0.01481066644191742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,1023,0.019354666272799175
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,1023,0.021141332884629566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,2047,0.027429332335789997
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,4095,0.046341334780057274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,4095,0.050842667619387306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,8191,0.08130666613578796
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,16383,0.15108799934387207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,8191,0.09220266342163086
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,2047,0.031445334355036415
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,16383,0.184063990910848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,511,0.2850773334503174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,32767,0.2876746654510498
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,32767,0.3508373498916626
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,65535,0.5668426752090454
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1,0.011871999750534693
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,float16,1023,0.7407999833424886
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,65535,0.6776533126831055
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,float16,131071,1.1288853486378987
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,2,2,128,1,float16,fp8,1023,0.4517066478729248
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,1,0.012527999778588613
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,3,0.011642667154471079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,3,0.012634667257467905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,7,0.012741333494583765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,15,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,1,128,1,float16,fp8,131071,1.3484479586283367
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,31,0.01192533348997434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,127,0.011882666498422623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,31,0.012869333227475485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,63,0.011813333878914515
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,255,0.015205333630243937
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,63,0.012784000486135483
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,127,0.01431999976436297
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,255,0.013503999759753546
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,7,0.01201066623131434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,511,0.015157333264748255
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,511,0.017456000049908955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,1023,0.01985599969824155
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,15,0.012655999511480331
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,1023,0.022005334496498108
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,2047,0.02863466739654541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,16383,0.1562346617380778
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,2047,0.03183999905983607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,4095,0.0495306650797526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,4095,0.052282666166623436
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,8191,0.0844586690266927
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,8191,0.09884799520174663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,16383,0.1808906594912211
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,32767,0.29190399249394733
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,32767,0.34649066130320233
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,65535,0.5720106760660807
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,65535,0.682528018951416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1,0.01179733375708262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,float16,131071,1.1387733618418376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,1,0.012341332932313284
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,15,0.011722666521867117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,2,2,128,1,float16,fp8,131071,1.3370399475097656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,3,0.012122667084137598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,3,0.012351999680201212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,7,0.011711999773979187
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,31,0.012453333785136541
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,7,0.012586666891972223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,15,0.01239466667175293
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,31,0.012026666353146235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,255,0.013866666704416275
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,63,0.012117333710193634
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,63,0.012250666817029318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,127,0.011920000116030375
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,127,0.013722666849692663
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,255,0.015087999403476715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,511,0.015557333827018738
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,511,0.01717866708834966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,1023,0.019941333681344986
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,1023,0.0215786670645078
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,2047,0.02959999938805898
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,2047,0.03159466634194056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,4095,0.048986668388048805
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,4095,0.05356266597906748
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,8191,0.08480532964070638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,32767,0.3514719804128011
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,8191,0.0995093286037445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,16383,0.15570666392644247
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,16383,0.1836479902267456
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,32767,0.29025065898895264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,65535,0.5721280177434286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,65535,0.6792480150858561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,3,0.011941333611806234
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1,0.011157333850860596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,1,0.01228800043463707
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,3,0.010853332777818045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,float16,131071,1.1449066797892253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,7,0.011087999989589056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,32,1,1,1,128,1,float16,fp8,131071,1.3562933603922527
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,7,0.012266666938861212
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,63,0.011909333368142446
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,15,0.011029332876205444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,127,0.010933333386977514
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,15,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,31,0.011322667201360067
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,31,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,63,0.011045332998037338
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,1023,0.014015999933083853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,127,0.014885333677132925
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,255,0.01331199953953425
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,255,0.014256000518798828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,511,0.013461332768201828
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,511,0.01451733335852623
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,8191,0.017946666727463405
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,1023,0.014554666976133982
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,2047,0.014842666685581207
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,16383,0.024634666740894318
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,32767,0.031925333042939506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,2047,0.016117333124081295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,4095,0.015770666301250458
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,4095,0.016602666427691776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,8191,0.019466667125622433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,16383,0.022426667312781017
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1,0.011178666104873022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,32767,0.034714666505654655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,65535,0.05003733436266581
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,65535,0.05552533268928528
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,float16,131071,0.08943466345469157
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1,1,1,1,128,1,float16,fp8,131071,0.09827199578285217
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,15,0.01090666651725769
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,15,0.01251199965675672
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,1,0.012047999848922094
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,3,0.010965333630641302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,3,0.011727999895811081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,7,0.010874666273593903
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,7,0.011973333855470022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,31,0.010821333775917688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,31,0.012272000312805176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,63,0.011055999745925268
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,63,0.01210133358836174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,127,0.010922666639089584
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,127,0.014453332871198654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,255,0.01370666672786077
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,255,0.014287999520699183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,511,0.01349866638580958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,511,0.014709333578745524
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,1023,0.014864000181357065
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,1023,0.01580799991885821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,2047,0.015989333391189575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,2047,0.016602666427691776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,4095,0.018170667191346485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,32767,0.048101335763931274
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,4095,0.01926933353145917
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,8191,0.021146667500336964
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,8191,0.023754666248957317
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,16383,0.030069333811601002
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,16383,0.03429333368937174
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,32767,0.05267733335494995
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,65535,0.08401599526405334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,65535,0.0939359962940216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,float16,131071,0.1518400013446808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2,1,1,1,128,1,float16,fp8,131071,0.18281600872675577
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,1,0.013376000026861826
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,3,0.012757333616415659
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,15,0.013381333400805792
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,3,0.01341333364446958
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,7,0.012671999633312225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,7,0.01309866706530253
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,15,0.012762666990359625
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,31,0.012693333129088083
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,255,0.014362666755914688
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,31,0.013424000392357508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,63,0.012522666404644648
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,63,0.013280000537633896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,127,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,1023,0.020949333906173706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,127,0.014783999572197596
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,255,0.015333333363135656
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,511,0.01613866661985715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,511,0.017973333597183228
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,1023,0.02257599929968516
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,2047,0.03781333317359289
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,2047,0.033717334270477295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,4095,0.06099733213583628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,4095,0.05831466615200043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,8191,0.10654399792353313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,8191,0.10128532846768697
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,16383,0.2025066614151001
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,16383,0.18480000893274942
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,32767,0.39213331540425617
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,32767,0.35547200838724774
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1,0.011178666104873022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,65535,0.7037279605865479
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,1,0.011642667154471079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,3,0.010938666760921478
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,65535,0.6883947054545084
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,15,0.01062400018175443
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,3,0.0122079998254776
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,float16,131071,1.527392069498698
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,31,0.011968000481526056
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,31,0.010853332777818045
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,7,0.010981333752473196
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,7,0.011519999553759893
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,64,1,1,1,128,1,float16,fp8,131071,1.3562560081481934
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,15,0.011407999942700068
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,63,0.011226666470368704
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,127,0.010597333312034607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,511,0.015311999867359797
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,127,0.014490666488806406
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,63,0.012037333101034164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,255,0.013370666652917862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,255,0.014159999787807465
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,511,0.01441066712141037
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,1023,0.016176000237464905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,1023,0.016250666230916977
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,2047,0.016810666769742966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,2047,0.01907733331123988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,4095,0.020928000410397846
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,4095,0.023584000766277313
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,8191,0.029824001093705494
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,8191,0.033717334270477295
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,16383,0.048112000028292336
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,16383,0.0525546669960022
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,32767,0.08376000324885051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,32767,0.0939466655254364
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,65535,0.15190399686495462
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,65535,0.18347734212875366
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,3,0.013728000223636627
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,float16,131071,0.29123733441034955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,4,1,1,1,128,1,float16,fp8,131071,0.3535360097885132
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,7,0.014149333039919535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1,0.014053333550691605
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,1,0.014896000425020853
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,3,0.015034666905800501
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,7,0.014736000448465347
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,31,0.014149333039919535
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,15,0.013663999736309052
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,15,0.014661333213249842
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,31,0.015109332899252573
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,63,0.013893333574136099
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,63,0.014943999548753103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,127,0.014741333822409311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,127,0.016437333077192307
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,255,0.01600533351302147
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,255,0.01714666684468587
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,511,0.018245333184798557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,511,0.019472000499566395
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,1023,0.036757332583268486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,1023,0.025765334566434223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,2047,0.05738133192062378
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,2047,0.04252266883850098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,8191,0.18532266219456991
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,4095,0.10037333766619365
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,8191,0.11349333326021831
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,4095,0.06714666883150737
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,16383,0.20600533485412598
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,16383,0.35527467727661133
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1,0.019466667125622433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,32767,0.6960960229237875
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,32767,0.38558932145436603
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,1,0.020954666038354237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,3,0.01979200045267741
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,3,0.02094399929046631
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,7,0.019424000134070713
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,float16,65535,1.480288028717041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,128,1,1,1,128,1,float16,fp8,65535,0.7508160273234049
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,15,0.019786667078733444
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,7,0.02067199970285098
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,31,0.019610666980346043
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,31,0.02082666630546252
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,15,0.021210665504137676
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,127,0.020090666910012562
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,127,0.02348800003528595
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,255,0.02380799998839696
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,255,0.025216000775496166
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,511,0.04218133290608724
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,511,0.031632001201311745
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,63,0.019487999379634857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,1023,0.06298133234182994
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,1023,0.048698668678601585
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,63,0.020954666038354237
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,8191,0.357807993888855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,2047,0.10444266597429912
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,2047,0.07400000095367432
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,8191,0.21196800470352173
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1,0.030928000807762146
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,4095,0.1893706719080607
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,4095,0.11866666873296101
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,1,0.03226666649182638
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,float16,16383,0.6918400128682455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,256,1,1,1,128,1,float16,fp8,16383,0.39190399646759033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,3,0.03198933353026708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,3,0.031023999055226643
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,7,0.03089066594839096
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,15,0.03236799935499827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,63,0.030821333328882854
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,127,0.033573334415753685
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,63,0.03215466688076655
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,31,0.03127466638882955
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,127,0.036346666514873505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,31,0.03246400008598963
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,7,0.031856000423431396
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,255,0.044010668992996216
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,15,0.030949334303538006
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,255,0.04878933231035868
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,511,0.07048533360163371
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,511,0.0605973352988561
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,1023,0.11221866806348164
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,1023,0.08357333143552144
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,8191,0.6942293643951416
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1,0.010879999647537867
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,3,0.01179733375708262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,3,0.010885333021481832
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,2047,0.19322667519251505
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,7,0.010762666662534079
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,1,0.012074666718641916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,7,0.011839999506870905
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,2047,0.13099732995033264
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,15,0.011765333513418833
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,31,0.011146667102972666
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,31,0.011834666132926941
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,63,0.0116799995303154
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,8191,0.40518399079640705
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,127,0.011253333340088526
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,127,0.013072000195582708
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,255,0.012063999970753988
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,511,0.016282666474580765
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,511,0.01515199989080429
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,1023,0.016970666746298473
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,1023,0.018858666221300762
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,2047,0.021418665846188862
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,2047,0.023749334116776783
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,4095,0.029674666623274486
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,4095,0.03369066615899404
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,255,0.015482666591803232
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,8191,0.04770133395989736
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,8191,0.05277333160241445
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,16383,0.08443733056386311
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,fp8,4095,0.2206559975941976
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,16383,0.09754666686058044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,32767,0.15181333820025125
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,15,0.01109333336353302
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,32767,0.18529067436854044
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,65535,0.2917120059331258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,63,0.01108266661564509
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,float16,131071,0.5675040086110433
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,131071,0.6795252958933512
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,512,1,1,1,128,1,float16,float16,4095,0.35979731877644855
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,8,1,1,1,128,1,float16,fp8,65535,0.350053350130717
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1,0.03723733375469843
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,1,0.03774933268626531
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,3,0.037962667644023895
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,3,0.03752533346414566
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,7,0.03733866661787033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,7,0.03782933453718821
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,15,0.03743999948104223
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,31,0.03713600089152654
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,31,0.037818667789300285
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,63,0.03862400104602178
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,63,0.03835733234882355
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,127,0.04535466432571411
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,127,0.048021331429481506
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,15,0.03770133356253306
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,255,0.0626933326323827
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,255,0.06257600088914235
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,511,0.1029813289642334
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,511,0.08614400029182434
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,1023,0.18599466482798258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1,0.06392000118891399
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,1023,0.12869866689046225
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,1,0.0651039977868398
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,2047,0.22447999318440756
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,2047,0.3593706687291463
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,3,0.06444266438484192
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,3,0.06465599934260051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,fp8,4095,0.40372268358866376
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,7,0.06414933502674103
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,1024,1,1,1,128,1,float16,float16,4095,0.7016533215840658
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,7,0.06537599861621857
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,31,0.06709866722424825
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,31,0.06519466638565063
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,15,0.06469866633415222
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,15,0.06436799963315327
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,63,0.06757333377997081
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,255,0.11069333553314209
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,255,0.1106666624546051
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,127,0.08808533350626628
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,511,0.15763200322786966
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,511,0.19187732537587485
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,63,0.07091733316580455
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,3,0.011328000575304031
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,3,0.012191999703645706
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,7,0.011194666226704916
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,7,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,15,0.011381333072980246
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,15,0.012154666086037954
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,1,0.011999999483426413
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,31,0.011231999844312668
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,31,0.01181866725285848
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1,0.01145600030819575
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,63,0.011125333607196808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,63,0.01179733375708262
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,127,0.011546666423479715
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,127,0.0751039981842041
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,127,0.013178666432698568
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,255,0.01257066677014033
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,255,0.01414399966597557
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,511,0.014581333845853806
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,1023,0.019173332800467808
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,1023,0.021327999730904896
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,1023,0.35660799344380695
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,2047,0.027466667195161183
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,511,0.016704000532627106
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,2047,0.031189332405726116
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,4095,0.045882667104403176
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,4095,0.05026666820049286
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,8191,0.09204266468683879
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,16383,0.14964266618092856
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,32767,0.28828267256418866
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,16383,0.18344533443450928
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,32767,0.3495733340581258
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,65535,0.5612800121307373
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,65535,0.6838560104370117
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,1023,0.24304533004760742
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,131071,1.1195039749145508
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,fp8,131071,1.345402717590332
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,16,1,1,1,128,1,float16,float16,8191,0.08115200201670329
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,float16,2047,0.7084533373514811
VLLM,0.14.0,NVIDIA H100 80GB HBM3,generation_attention,vllm_flash_attn,2048,1,1,1,128,1,float16,fp8,2047,0.433296004931132
