framework,version,device,op_name,kernel_source,gemm_dtype,m,n,k,latency
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,8192,18.587799072265625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,10240,22.905229356553818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,12288,27.217117309570312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,8192,31.739278157552082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,16384,35.85681491427952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,10240,40.53974405924479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,7168,16.470130072699654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,12288,47.9828626844618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,7168,27.845704820421005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,6144,15.9223141140408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,6144,23.824981689453125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,5120,19.960010104709202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,5120,13.253072102864584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,4096,16.039832221137154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,16384,61.75987413194445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,4096,10.551732381184896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,3584,10.022071838378906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,3072,8.544905768500433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,3584,13.921887715657553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,3072,11.941468132866753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,2560,7.616260104709202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,2048,6.558021121554905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,2560,10.513075934516058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,2048,7.989850362141927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,1536,8.11091783311632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,1024,4.9917522006564665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,1536,5.18578126695421
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,512,3.569919162326389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,1024,4.309893290201823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,768,4.573472764756945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,768,3.887366400824653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,256,2.342767079671224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,128,2.179478327433268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,256,3.0719956292046438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,64,2.0525218115912542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,65536,32,2.4912711249457464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,64,2.6182329389784074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,128,2.874775144788954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,512,3.492466608683268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,65536,32,2.6264862484402127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,10240,8.605928209092882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,10240,9.989924960666233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,12288,10.069237603081598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,12288,12.131823221842447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,16384,13.11134507921007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,16384,15.880839029947916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,8192,6.5543458726671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,7168,7.474728902180989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,8192,8.480428059895834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,7168,5.791522555881077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,6144,5.988362630208333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,6144,5.062970479329427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,4096,3.6238488091362844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,5120,5.806952158610026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,5120,4.5167316860622835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,4096,5.098319159613715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,3584,4.1138916015625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,3584,3.34245596991645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,2048,2.030544916788737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,3072,3.91518063015408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,2560,2.446054246690538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,2560,3.180034637451172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,3072,2.8767066531711154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,2048,1.9160719977484808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,1536,1.4736933178371852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,768,1.1763377719455295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,1024,1.3045315212673612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,1024,1.539601749844021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,768,1.0633217493693035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,1536,1.581007957458496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,512,0.6943644417656792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,128,0.3889377646976047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,256,0.46981509526570636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,64,0.3953368928697374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,128,0.7440106603834363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,32,0.3792773352728949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,256,0.9295768737792969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,512,1.0950933032565646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,64,0.667916456858317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,32,0.7320382330152723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,12288,9.012932671440973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,12288,8.430562337239584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,16384,11.117318895128038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,16384,12.177557203504774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,10240,7.4788911607530375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,10240,7.145168728298611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,8192,5.8367199367947045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,8192,8.63635508219401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,7168,5.317685445149739
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,7168,5.141316307915582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,16384,65536,53.01391262478299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,6144,6.5789672003851996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,6144,4.412201775444879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,5120,3.7355779012044272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,5120,5.415642632378472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,3584,2.648102230495877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,4096,4.01546393500434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,4096,3.1217503017849393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,3584,2.69270748562283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,3072,2.3684285481770835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,2560,2.0151074727376304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,2560,2.2672932942708335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,3072,3.2441351148817272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,2048,1.8909848531087239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,16384,65536,61.87994384765625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,2048,1.5509707132975261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,1536,1.52765867445204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,1024,1.2178079817030165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,1536,1.2335093816121419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,1024,0.9451404147677951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,256,0.3505742284986708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,768,0.8665342330932617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,512,0.6390693452623155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,128,0.28767289055718315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,768,0.9387430614895291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,512,0.720545768737793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,64,0.281400892469618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,256,0.6006337801615397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,32,0.31014400058322483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,64,0.5126613510979546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,128,0.6569075584411621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,32,0.5028933419121636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,12288,7.786122639973958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,12288,7.599129570855035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,16384,10.058542887369791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,16384,10.13736809624566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,12288,65536,45.012166341145836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,12288,65536,46.517673068576386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,10240,6.2752532958984375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,10240,6.484673817952474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,8192,5.261015997992621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,7168,4.473906622992621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,8192,6.873643663194444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,7168,4.56655036078559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,6144,4.009581247965495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,5120,3.158043543497721
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,6144,4.010612487792969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,5120,3.4525716569688587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,4096,3.349100536770291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,4096,2.8017039828830295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,3072,2.0159564548068576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,3584,2.933831956651476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,3584,2.4540186987982855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,2560,1.5399022632175023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,3072,2.128779517279731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,2048,1.2414337793986003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,2560,1.8953787485758464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,2048,1.4338106579250758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,1536,1.2047920227050781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,1024,0.8841404385036893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,1536,1.069664849175347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,768,0.5832915306091309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,512,0.4904799991183811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,256,0.2491955492231581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,1024,0.8596435652838813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,768,0.7223715782165527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,128,0.24974489212036133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,64,0.2451102203792996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,512,0.6970444255405002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,256,0.579894224802653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,32,0.23657600084940592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,128,0.497977786593967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,64,0.4307075606452094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,32,0.4224444495307074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,12288,6.052003648546006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,16384,8.197725084092882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,12288,7.450096978081597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,16384,9.79251946343316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,10240,5.024792989095052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,8192,4.045539432101779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,10240,65536,38.78884548611111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,10240,6.260656992594401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,8192,5.110322740342882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,7168,4.866747538248698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,7168,4.494152069091797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,10240,65536,41.05563693576389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,6144,3.081485324435764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,5120,2.57371457417806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,6144,3.976787567138672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,4096,2.0531298319498696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,5120,3.38447994656033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,3584,2.2083511352539062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,4096,2.955908457438151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,3584,2.58604007297092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,3072,2.0224363538953996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,2560,1.6179929309421117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,3072,2.2851778666178384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,2048,1.1422995461357963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,1536,1.126190185546875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,1024,0.5946524408128526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,2560,2.00619019402398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,2048,1.6028231514824762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,768,0.6029582023620605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,512,0.3109671009911431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,1536,1.319734255472819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,256,0.23261332511901855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,1024,1.162642690870497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,128,0.19708177778455946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,64,0.20005867216322157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,768,1.0586559507581923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,256,0.8915742238362631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,32,0.21295289198557535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,512,0.9977501763237847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,128,0.8552133242289225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,64,0.8021519978841146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,32,0.8133164511786567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,12288,6.040105607774522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,8192,65536,31.209398057725693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,16384,7.2804718017578125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,12288,7.029765235053168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,16384,9.237742953830296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,10240,4.902724372016059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,8192,4.9612346225314665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,8192,65536,35.66557820638021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,7168,3.5580664740668406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,10240,5.9829529656304254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,8192,4.805344899495442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,6144,2.64649412367079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,5120,2.148952907986111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,7168,4.278921763102214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,6144,3.8340526156955295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,4096,2.1334826151529946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,5120,3.2351076338026257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,3584,2.1094373067220054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,4096,2.6693492465549045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,3072,1.7419759962293837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,2560,1.4736720191107855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,3584,2.3942453596327042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,2048,0.8667582405938042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,3072,2.1524747212727866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,1024,0.544090641869439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,1536,0.8534017668830024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,2560,1.831833733452691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,2048,1.4659350713094075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,1536,1.1885279549492729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,768,0.4203182326422797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,512,0.27406220965915257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,256,0.16998399628533256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,1024,0.9891199535793729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,128,0.16618311405181885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,64,0.17864089541965059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,768,0.9485324223836263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,512,0.8492115338643392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,32,0.18840621577368843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,256,0.7859182357788086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,128,0.7363991207546658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,64,0.7028649118211535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,32,0.7188693152533637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,12288,4.5337982177734375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,16384,6.041354709201389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,7168,65536,27.2191399468316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,12288,6.643921746148004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,10240,3.7982177734375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,16384,8.657890319824219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,8192,4.392313639322917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,7168,2.6078853607177734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,10240,5.606436411539714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,8192,4.539082845052083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,6144,2.311552047729492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,7168,65536,33.78315904405382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,7168,4.022037506103516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,5120,2.5688239203559027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,3584,1.3588746388753254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,4096,2.1362853580050998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,5120,3.0342445373535156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,6144,3.5421572791205516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,4096,2.6256514655219183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,3072,1.3633421791924372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,2560,1.0038302739461262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,3584,2.2339945899115667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,2048,0.683869308895535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,1536,0.6242213249206543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,2560,1.698386616177029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,1024,0.40907732645670575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,3072,2.006410598754883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,2048,1.3218773735894096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,512,0.2578364478217231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,768,0.3638177712758382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,256,0.14866311020321316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,1536,1.0668907165527344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,1024,0.8997369342380099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,128,0.1530524492263794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,768,0.8484569125705295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,64,0.1496257781982422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,32,0.15839466783735487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,512,0.7463688850402832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,256,0.6741609043545194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,128,0.6325573391384548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,64,0.6336151229010688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,32,0.6177333196004232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,12288,3.8244684007432728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,16384,5.155047946506076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,6144,65536,24.624513414171005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,12288,6.370369805230035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,10240,3.1971698337131076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,8192,2.5570852491590714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,16384,8.315419514973959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,7168,2.2085982428656683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,10240,5.346370697021484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,6144,2.1821778615315757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,8192,4.317530737982856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,7168,3.8142729865180125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,5120,2.122751024034288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,6144,3.3139964209662547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,6144,65536,32.00177001953125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,4096,1.5671990712483723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,5120,2.822571648491753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,3072,0.8969039916992188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,3584,1.4912808736165364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,4096,2.354535208808051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,2560,0.9988684124416776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,3584,2.057098600599501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,2048,0.7022417916191949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,3072,1.8178560468885634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,1024,0.44542397393120664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,1536,0.5973519749111599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,2560,1.5573333104451497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,2048,1.1621884240044487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,512,0.19952532980177137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,768,0.4077315595414903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,1536,0.9375431272718641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,256,0.1194266610675388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,128,0.12722578313615587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,1024,0.7643831041124133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,768,0.7185653580559624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,64,0.12132178412543403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,32,0.12640533182356092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,512,0.6290213267008463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,256,0.5935662057664659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,128,0.5262542300754124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,64,0.5054053200615777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,32,0.5017875565422906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,5120,65536,19.698964436848957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,12288,3.0527848137749563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,16384,4.052735222710504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,10240,2.6725529564751516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,12288,5.545441097683376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,8192,2.0165466732449002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,16384,7.353406270345052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,10240,4.666951073540582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,7168,2.3872222900390625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,8192,3.8192952473958335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,6144,1.500322659810384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,7168,3.3418933020697703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,5120,1.5928612815009222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,4096,1.0082595613267686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,6144,2.876830206976997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,3584,0.9900080362955729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,5120,2.429413265652127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,4096,1.955947452121311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,3584,1.714598231845432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,3072,0.8443102306789823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,2560,0.6967760192023383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,2048,0.5619635581970215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,1536,0.4892435603671604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,3072,1.4838746388753254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,1024,0.3270755608876546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,2048,0.8941911061604818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,2560,1.2243341869778104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,768,0.24451200167338052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,1536,0.6349715656704372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,1024,0.4363128874037001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,512,0.1589555607901679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,256,0.0873875551753574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,768,0.386693345175849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,128,0.04385066694683499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,5120,65536,31.005320231119793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,512,0.33614667256673175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,64,0.07446666558583577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,256,0.23085599475436738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,128,0.20518933402167427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,64,0.18334221839904785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,32,0.0358168880144755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,32,0.1910720003975762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,4096,65536,16.85194566514757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,12288,2.7943592071533203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,16384,3.550047980414497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,8192,1.7443129221598308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,10240,2.8600790235731335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,12288,5.405647277832031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,16384,7.114178975423177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,7168,2.2209129333496094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,10240,4.512299431694879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,6144,1.2999031278822157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,5120,1.1023972829182942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,7168,3.2183840009901257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,8192,3.6682010226779513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,4096,0.8893671035766602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,3584,0.7791635725233289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,6144,2.802141401502821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,3072,0.6752026875813802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,4096,1.8744587368435328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,5120,2.3372436099582248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,2560,0.5776444541083442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,3584,1.6553084055582683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,2048,0.49104801813761395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,1536,0.3215324348873562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,2048,0.7942577997843424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,3072,1.483290672302246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,2560,1.1694524553087022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,1024,0.23101509941948783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,1536,0.6010675430297852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,768,0.20990310774909127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,512,0.14426310857137045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,1024,0.4028613302442763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,256,0.07980177799860637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,768,0.34926578733656144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,128,0.04093599981731839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,512,0.27576621373494464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,64,0.03789955708715651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,256,0.20609777503543428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,32,0.029983109898037378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,128,0.18499467107984754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,64,0.16600178347693548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,32,0.16216354899936253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3584,65536,14.604153103298612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,4096,65536,29.281246609157986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,12288,2.746283637152778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,16384,3.7170015970865884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,10240,1.9725057813856337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,8192,1.6875875261094835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,7168,1.8533804151746962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,16384,6.947833591037327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,12288,5.184651692708333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,8192,3.539720959133572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,10240,4.391081068250868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,6144,1.0796978208753798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,5120,0.9363369411892362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,4096,0.8286515341864692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,3584,0.6801564428541395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,6144,2.6688586341010203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,7168,3.1600863138834634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,5120,2.237184100680881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,3072,0.5130346616109213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,4096,1.8162657419840496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,2560,0.5065920088026259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,2048,0.4055688911014133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,3584,1.5698578092787001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,1536,0.33232800165812176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,65536,12.168136596679688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,3072,1.3403893576727972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,2560,1.1094257566663954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,1024,0.2384613355000814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,512,0.12584711445702448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,768,0.18481600284576416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,2048,0.7868249151441785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,256,0.07245422071880765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,128,0.03816533419820998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,768,0.3064311080508762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,64,0.03185955683390299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,512,0.28714577356974286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,1024,0.3933146794637044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,1536,0.5319369104173448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,3072,32,0.027757333384619817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,256,0.18450933032565645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,128,0.1694400045606825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,64,0.1410586701499091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,32,0.14471466011471218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,12288,1.9295448727077906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3584,65536,28.26877170138889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,16384,2.874725341796875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,10240,1.597315576341417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,8192,1.337265756395128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,7168,1.1029848522610135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,12288,5.1007465786404085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,6144,1.1631786558363173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,16384,6.761785719129775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,10240,4.345678117540148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,8192,3.4227884080674915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,5120,0.8556604385375977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,4096,0.6780213250054253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,7168,3.015109380086263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,3584,0.6665253109402126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,3072,0.5443920029534234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,5120,2.160302268134223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,6144,2.599649853176541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,4096,1.7591688368055556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,2560,0.4177422258589003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,3584,1.5048240025838215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,2048,0.2923271126217312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,1536,0.25596798790825737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,2048,0.7057991027832031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,1024,0.171926220258077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,3072,1.2895751529269748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,1536,0.4622853597005208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,768,0.14322488837771946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,2560,1.0501057306925456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,512,0.09802310996585423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,256,0.07002844413121541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,1024,0.3440888987647162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,512,0.23189687728881836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,128,0.029633777009116277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,768,0.2786453300052219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,256,0.15445155567593044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,64,0.02804888950453864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,32,0.02141777839925554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,128,0.1360026730431451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,64,0.12038132879469131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,32,0.12071555190616184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2560,65536,11.952845255533854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,12288,1.884169684516059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,16384,2.1644195980495877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,3072,65536,27.382576836480037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,10240,1.36572085486518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,8192,0.9562080171373156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,7168,0.898596445719401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,16384,6.503671010335286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,12288,4.931266784667969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,8192,3.3458169301350913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,10240,4.113324483235677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,6144,0.8181164529588488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,5120,0.6236053572760688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,4096,0.6337813271416558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,7168,2.9080621931287975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,3584,0.5708470874362522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,3072,0.48381953769259983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,5120,2.0776568518744574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,4096,1.6743208567301433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,6144,2.5266604953342013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,65536,10.096179538302952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,2560,0.2982204490237766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,2048,0.23600355784098306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,3584,1.457008891635471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,1536,0.21249155203501383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,1024,0.12613956133524576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,768,0.09907288683785333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,1024,0.2829893430074056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,1536,0.42863112025790745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,512,0.07459466987186007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,2560,1.0178569157918294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,2048,0.7142417695787219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,256,0.04468889037768046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,768,0.2555617756313748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,128,0.029122667180167303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,3072,1.277414215935601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,512,0.18927378124660918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,64,0.02200711104604933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,2048,32,0.017678222722477384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,256,0.13040088282691106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,128,0.11821777290768093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,64,0.10525421963797675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,32,0.09843733575608994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,12288,1.2996560202704537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,16384,1.637937757703993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,10240,1.102456940544976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,8192,1.0401350657145183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2560,65536,26.80823262532552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,12288,4.716426849365234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,7168,0.6136630905999078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,10240,3.9542499118381076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,6144,0.5196391211615669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,16384,6.226823170979817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,5120,0.5452115800645616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,8192,3.184008068508572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,4096,0.480621337890625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,3584,0.3829386764102512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,7168,2.7927572462293835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,65536,8.401019626193577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,3072,0.24999022483825684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,6144,2.4082319471571183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,2560,0.2110959953731961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,4096,1.5941990746392145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,2048,0.19583466317918566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,5120,2.0030595991346574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,1536,0.15853244728512234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,3584,1.400144894917806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,1024,0.11007289091746013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,768,0.0878560013241238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,1536,0.3899128966861301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,2048,0.5961786905924479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,1024,0.24311200777689615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,2560,0.9607653088039823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,512,0.07177866829766168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,256,0.03369777732425266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,768,0.19346933894687227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,3072,1.1907546785142686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,128,0.02046133412255181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,32,0.014128888646761576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1536,64,0.01761155492729611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,512,0.15405333042144775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,128,0.09109244743982951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,256,0.10493866602579753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,32,0.07952711317274305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,64,0.08136977752049764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,12288,0.6989288859897189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,16384,1.1156258053249783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,10240,0.6534950998094347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,8192,0.5469475322299534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,12288,4.524984995524089
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,2048,65536,25.613120185004338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,10240,3.8002586364746094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,65536,5.422754499647353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,7168,0.38730043835110134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,8192,3.051816092597114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,6144,0.3318275610605876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,5120,0.3085422250959608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,4096,0.24496089087592232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,3584,0.23253154754638672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,16384,5.99031745062934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,3072,0.17595999770694307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,4096,1.5174666510687933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,3584,1.3279679616292317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,5120,1.9085583157009547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,2560,0.14571289221445718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,6144,2.316513697306315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,2048,0.11729421880510117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,1536,0.0906568898095025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,7168,2.8690293629964194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,1024,0.06433333290947808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,2048,0.5381991068522135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,1536,0.35152533319261337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,3072,1.11973328060574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,768,0.05617955658170912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,512,0.03842666745185852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,2560,0.9111671447753906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,256,0.02239822182390425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,1024,0.20654400189717612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,768,0.1511039998796251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,128,0.014647111296653748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,512,0.10323822498321533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,64,0.011770666473441653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,1024,32,0.00998222248421775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,256,0.07610578007168241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,128,0.06381244129604764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,64,0.05829777982499865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,32,0.057625777191585965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,12288,0.5679839981926812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,16384,0.7883377605014377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,10240,0.5603297551472982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,8192,0.5088399781121148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,65536,4.228505028618707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,7168,0.4578897688123915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,10240,3.7183439466688366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,12288,4.449917263454862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,8192,2.984557257758247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,6144,0.38858665360344785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,5120,0.31944089465671116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,4096,0.2598408857981364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,16384,5.894502427842881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1536,65536,24.874376085069443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,3584,0.1954417758517795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,3072,0.17425421873728433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,6144,2.2493040296766496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,5120,1.8797164493136937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,4096,1.4880613750881617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,7168,2.618849860297309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,2560,0.15200444062550864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,2048,0.1295902199215359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,1536,0.08083555433485243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,1024,0.06807288858625624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,3072,1.088271141052246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,1536,0.3278782102796766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,768,0.04081066780620151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,1024,0.2166222201453315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,3584,1.3786658181084528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,512,0.029678222205903795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,2048,0.5141066445244683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,256,0.0181688881582684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,128,0.012424888710180918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,256,0.06279022163814969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,512,0.09028355280558269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,768,0.15831555260552302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,2560,0.8793964385986328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,64,0.010113777385817634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,768,32,0.00849955528974533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,64,0.04677155613899231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,128,0.052546666728125684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,32,0.044610665904151074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,16384,0.5477511088053385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,12288,0.432695123884413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,10240,0.40866756439208984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,8192,0.33307022518581814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,65536,2.7563838958740234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,7168,0.29800089200337726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,8192,2.921489715576172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,10240,3.642638312445747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,6144,0.19873866770002577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,12288,4.3558837042914496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,5120,0.1490186717775133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,4096,0.1279911067750719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,7168,2.566080093383789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,3584,0.101565334531996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,16384,5.807104746500651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,3072,0.09208889140023126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,4096,1.4503421783447266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,2560,0.07465689049826728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,5120,1.8272471957736547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,1024,65536,23.74963887532552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,6144,2.204763624403212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,2048,0.06461955441368951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,1536,0.046979556481043495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,1024,0.034675555096732244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,3584,1.2576168908013237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,768,0.028062221076753404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,1536,0.3137386639912923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,1024,0.19902400175730386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,512,0.019857777489556205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,768,0.13603466086917454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,256,0.012477333347002665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,128,0.00848355558183458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,2560,0.8636115392049154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,3072,1.0615040461222331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,512,0.07550577984915839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,2048,0.5658684306674534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,64,0.0069679998689227635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,256,0.049774222903781466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,64,0.035465776920318604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,128,0.04014666544066535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,512,32,0.006218666831652324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,32,0.0332551101843516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,16384,0.45987643135918516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,12288,0.36935112211439347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,10240,0.30530399746365017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,65536,1.6945671505398219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,8192,0.25573333104451496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,7168,0.19969777266184488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,10240,3.604437510172526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,6144,0.15937866104973689
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,12288,4.3382241990831165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,8192,2.8871485392252603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,7168,2.523140377468533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,5120,0.08032355705897014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,4096,0.060808002948760986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,3584,0.0566177765528361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,16384,5.7656097412109375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,3072,0.047712001535627574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,3584,1.21615113152398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,4096,1.406038178337945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,2560,0.03941333293914795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,2048,0.03168977631462945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,5120,1.7907625834147136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,1536,0.02569599946339925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,6144,2.1557324727376304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,768,65536,23.21670193142361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,3072,1.028305795457628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,1024,0.01920088960064782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,768,0.01534755527973175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,2048,0.4922337532043457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,512,0.011111111276679568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,1536,0.30007023281521267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,256,0.007419555551475949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,2560,0.8363173272874621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,128,0.005511111269394557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,768,0.11062755849626328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,64,0.0046302220887608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,512,0.05515999926461113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,256,32,0.004072000169091755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,256,0.036329779360029436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,1024,0.19095822175343832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,128,0.027383999692069158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,64,0.02382933265633053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,32,0.02159022291501363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,12288,0.34469066725836855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,16384,0.48784176508585614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,10240,0.2656293445163303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,8192,0.2170808845096164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,65536,1.6483511394924586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,7168,0.2114879952536689
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,8192,2.862510257297092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,6144,0.09825066725413005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,10240,3.5815921359592013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,7168,2.5019698672824435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,12288,4.292372385660808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,5120,0.04580266608132256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,4096,0.04257333278656006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,3584,0.040878223048316106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,16384,5.731031205919053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,3072,0.03464444478352865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,4096,1.3884799745347765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,3584,1.1990221871270073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,2560,0.028985778490702312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,2048,0.021887110339270696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,5120,1.7641315460205078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,6144,2.131399154663086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,1536,0.018735999862353008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,1024,0.012647999657524956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,1024,0.12261511219872369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,1536,0.2668986585405138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,2048,0.4651653501722548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,768,0.010440000229411654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,512,0.008208889100286696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,256,0.0058337777025169795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,768,0.07185333304935031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,3072,1.0104231304592557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,512,0.045702222320768565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,128,0.00480266660451889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,2560,0.8162737952338325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,64,0.004013333469629288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,256,0.030969778696695965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,512,65536,22.815597534179688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,128,32,0.003702222059170405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,128,0.023194667365815904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,64,0.0195777780479855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,32,0.017440888616773818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,12288,0.34160176912943524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,16384,0.4316444396972656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,10240,0.25839466518825954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,8192,0.2103591097725762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,65536,1.6277039845784504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,7168,0.20119021998511422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,8192,2.8620444403754344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,10240,3.578147464328342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,6144,0.06021955278184679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,12288,4.295502132839627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,5120,0.045237332582473755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,4096,0.03218133250872294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,16384,5.731335110134548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,4096,1.387867503696018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,3584,0.03302755620744493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,3072,0.02767466670937008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,5120,1.7684675852457683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,6144,2.1399430168999567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,2560,0.02344622214635213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,2048,0.017322666115230985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,7168,2.5042008293999567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,1536,0.015166223049163818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,2048,0.4601270887586806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,1024,0.010050666828950247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,1536,0.2694568898942736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,3072,1.0159937540690105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,768,0.008756444685988957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,512,0.006032889087994893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,3584,1.2030275132921007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,1024,0.11933244599236383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,2560,0.8213982052273221
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,256,0.004621333132187526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,512,0.04558222161398994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,768,0.07249955336252849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,128,0.003920000046491623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,64,0.003125333330697484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,256,0.03042666779624091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,128,0.02294933299223582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,64,32,0.0029502221279674103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,64,0.019518221418062847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,32,0.017633777525689866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,256,65536,22.837015787760418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,16384,0.4072728951772054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,12288,0.3373226589626736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,10240,0.25718045234680176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,8192,0.21140889326731363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,65536,1.6290168762207031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,7168,0.18587822384304473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,8192,2.8585618336995444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,6144,0.0727528863483005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,10240,3.5844650268554688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,5120,0.03789688812361823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,7168,2.494823031955295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,4096,0.027779555983013574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,12288,4.4076008266872835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,3584,0.028304888142479792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,16384,5.7247814602322045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,4096,1.3856755362616644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,3072,0.024877332978778418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,3584,1.1974560419718425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,2560,0.02053244411945343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,5120,1.7647635142008464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,2048,0.015356444650226168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,1536,0.013027555412716336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,6144,2.13447994656033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,1024,0.009457777771684859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,1024,0.11668888727823894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,1536,0.2684222327338325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,768,0.007707555260923173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,512,0.005678222411208683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,2048,0.4765724605984158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,768,0.0705982247988383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,256,0.004487110922733943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,2560,0.8188586764865451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,512,0.04537333382500542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,128,0.003654222107595868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,3072,1.0082755618625217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,256,0.03042222062746684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,64,0.0028577778074476454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8192,32,32,0.002586666701568498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,64,0.01962933275434706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,128,0.02305511136849721
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,32,0.01751733322938283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,128,65536,22.682740953233505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,64,65536,22.560626559787327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,10240,11.836207919650606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,12288,13.698126898871529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8192,32,65536,22.734198676215275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,16384,17.989769829644096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,10240,20.275553385416668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,8192,9.361409505208334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,12288,24.262008666992188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,7168,8.936720954047308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,8192,16.7591552734375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,7168,14.053467644585503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,6144,7.916185167100694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,6144,12.117200215657553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,4096,5.299327002631293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,5120,6.174672868516709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,5120,9.917210049099392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,16384,32.48560926649306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,4096,7.983629862467448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,3584,4.947100321451823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,3584,6.96908442179362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,3072,4.340246412489149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,2560,3.4807618459065757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,2560,5.0965423583984375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,3072,6.079177008734809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,2048,4.026824951171875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,2048,3.2564320034450955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,1536,3.0831538306342234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,512,1.6015635596381292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,1024,2.085028542412652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,1536,2.69615109761556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,1024,3.4709883795844187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,768,2.0458621978759766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,768,1.8906675974527996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,512,1.7682328754001195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,256,0.9605066511366102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,64,1.0847999784681532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,128,1.092349370320638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,65536,32,1.1630676057603624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,256,1.477752897474501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,32,1.3421857621934679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,64,1.3280995686848958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,65536,128,1.4562515682644313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,10240,4.282070159912109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,10240,5.08238771226671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,12288,5.01692623562283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,12288,6.096031188964844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,16384,6.695526123046875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,16384,8.084849039713541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,8192,4.121090783013238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,8192,3.279464933607313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,7168,3.552021238538954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,7168,3.016006257798937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,6144,3.040760040283203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,6144,2.6958062913682728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,4096,1.9505271911621094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,4096,1.6425111558702257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,5120,2.122419569227431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,5120,2.5632425944010415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,3584,1.7411218219333227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,3584,1.4585928387112086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,3072,1.7035208808051214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,2048,1.144079950120714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,3072,1.24853515625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,2560,1.076141357421875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,2560,1.6153884463840062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,1024,0.5676177872551812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,1024,0.6176586680942112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,2048,0.9025733735826281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,768,0.5776444541083442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,1536,0.7959520022074381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,1536,0.8952835930718316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,512,0.33724178208245176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,768,0.6058977974785699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,256,0.24346489376491973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,512,0.47930579715304905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,128,0.19806488355000815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,256,0.4014711115095351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,64,0.19260533650716147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,32,0.2067484458287557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,128,0.42719199922349715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,64,0.3692302174038357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,32,0.4028008778889974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,12288,4.544317457411024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,12288,4.279049767388238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,16384,5.594298468695746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,16384,6.077032725016276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,10240,3.790805392795139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,10240,3.6210577223036022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,8192,2.8189830780029297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,8192,3.4484265645345054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,7168,2.6590667300754123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,7168,2.5763030582004123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,6144,2.2770489586724176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,6144,2.148047129313151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,16384,65536,26.507442898220486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,4096,1.4952764511108398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,4096,1.348515510559082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,5120,1.863783942328559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,5120,1.8645333184136286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,3584,1.8489803738064234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,3072,1.020433743794759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,3072,1.0059635374281142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,2560,0.9842453002929688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,3584,1.1767244338989258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,2560,0.8738471137152778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,2048,0.749660438961453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,1536,0.6155618031819662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,1024,0.4220115608639187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,1536,0.5984569125705296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,2048,0.7097599771287707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,512,0.26002576616075307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,768,0.365156438615587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,16384,65536,31.081654866536457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,256,0.15363644229041204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,1024,0.515020423465305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,768,0.463922659556071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,512,0.363663117090861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,32,0.15158932738833958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,128,0.15561421712239584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,64,0.1446053319507175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,128,0.32068000899420845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,256,0.34574045075310605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,64,0.27216890123155385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,32,0.2646160125732422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,12288,3.8036393059624563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,16384,5.086793687608507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,16384,5.187654283311632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,12288,65536,22.37057664659288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,12288,65536,23.393575032552082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,10240,3.1790915595160594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,12288,3.9293564690483938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,8192,2.826635572645399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,10240,3.28784539964464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,8192,2.6464434729682074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,6144,1.9056044684516058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,7168,2.6495653788248696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,7168,2.337043550279405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,6144,1.9557298024495442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,5120,1.5835662417941625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,4096,1.207383155822754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,4096,1.240438249376085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,5120,1.7326141993204753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,3584,1.0758035447862413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,3584,1.036368899875217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,3072,1.1222107145521376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,3072,0.8886897828843859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,2048,0.6773529052734375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,1536,0.609148449367947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,2560,0.7353510856628418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,2560,0.8522044287787544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,2048,0.6219475534227159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,768,0.31475644641452366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,1024,0.4416240056355794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,1536,0.5205555491977268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,512,0.21430577172173393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,768,0.4122328758239746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,1024,0.4468640221489801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,256,0.12376622358957927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,128,0.11907022529178196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,512,0.37012089623345273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,32,0.11632622612847222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,64,0.11914311514960395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,256,0.26429955164591473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,64,0.24071023199293348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,128,0.26240621672736275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,32,0.23974845144483778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,12288,3.055532455444336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,16384,4.077397240532769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,12288,3.8437037997775607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,16384,5.004766252305773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,10240,2.5464320712619357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,8192,2.0677547454833984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,10240,65536,19.66620551215278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,10240,3.2583368089463978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,7168,1.7820275624593098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,10240,65536,20.541097005208332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,8192,2.6112054189046225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,6144,2.0145742628309464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,7168,2.326765272352431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,5120,1.3320062425401475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,4096,1.10552978515625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,5120,1.7442311180962458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,3584,0.8551715744866265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,4096,1.3196631537543404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,6144,2.0089244842529297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,3072,0.9492888980441623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,2048,0.5581830872429742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,3584,1.1352888743082683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,2560,0.716209782494439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,1024,0.34738845295376247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,1536,0.4383591016133626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,2560,0.9053849114312066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,3072,1.0161795086330838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,768,0.22503556145562065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,2048,0.7734773423936633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,1536,0.6599929067823622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,512,0.1725191142823961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,128,0.06119644641876221
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,256,0.08722133106655544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,64,0.04325955443912082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,1024,0.5970142152574327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,768,0.5731804635789659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,32,0.07404355870352851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,512,0.48921775817871094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,256,0.4438684516482883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,64,0.41078665521409774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,128,0.43349067370096844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,32,0.43543110953436953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,12288,2.866590287950304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,8192,65536,15.860005696614584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,16384,3.613650851779514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,12288,3.607452392578125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,16384,4.7256215413411455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,10240,2.279913796318902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,8192,1.7932898203531902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,8192,65536,18.40221659342448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,10240,3.0374071333143444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,7168,1.626375092400445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,6144,1.4168311225043402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,8192,2.468263202243381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,7168,2.185092502170139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,4096,0.8664124276902941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,5120,1.3034044901529949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,4096,1.1929421954684787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,3584,0.6964195569356283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,5120,1.6041244930691188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,6144,1.9792266421847875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,3072,0.601570659213596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,2560,0.6078746583726671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,3584,1.153510199652778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,3072,0.9113840527004666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,2048,0.4890151023864746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,2560,0.7807662222120496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,1536,0.37447643280029297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,1024,0.28027023209465873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,768,0.222034666273329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,2048,0.6969413227505155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,512,0.15847022003597683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,1536,0.6082977718777126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,256,0.0923617747094896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,1024,0.5236364470587836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,128,0.05717333157857259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,64,0.04469866553942362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,512,0.43491821818881565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,768,0.5161226590474447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,32,0.04754933383729723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,256,0.3966222339206272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,128,0.3740035427941217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,64,0.36400000254313153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,32,0.3493493398030599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,7168,65536,13.956058926052519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,12288,2.835068384806315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,16384,3.138932545979818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,12288,3.3908807966444225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,8192,1.575598186916775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,16384,4.461956448025174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,10240,2.5159734090169272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,10240,2.8640978071424694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,7168,1.3435689078436956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,8192,2.2775431738959417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,6144,1.175904909769694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,7168,65536,17.44835917154948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,7168,2.0390737321641708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,5120,1.150793817308214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,5120,1.4752666685316298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,3584,0.7300604184468588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,6144,1.747288915846083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,4096,1.0137626859876845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,4096,1.0970373153686523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,3584,0.9768888685438367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,3072,0.6236675580342611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,2560,0.49995109770033097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,2048,0.41880000962151426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,3072,0.831889788309733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,1536,0.39190133412679035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,2560,0.8070249027676053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,1024,0.22572623358832467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,2048,0.6235413551330566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,768,0.19992621739705405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,512,0.12564710776011148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,1536,0.5368382136027018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,1024,0.4633466402689616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,256,0.08041510979334514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,128,0.04137422309981452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,64,0.037780443827311196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,32,0.03085244364208645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,768,0.4281795554690891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,512,0.38643021053738064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,256,0.34893155097961426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,128,0.32341599464416504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,64,0.30949510468377006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,32,0.3076275454627143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,12288,2.301300472683377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,16384,2.5761341518825955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,12288,3.189898596869575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,10240,1.6035298241509333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,6144,65536,14.594896104600695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,16384,4.183084275987413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,8192,1.4017306433783636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,10240,2.6768845452202688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,7168,1.482110235426161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,6144,1.0386924743652344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,8192,2.134030236138238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,7168,1.9055680168999567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,5120,0.8995431264241537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,6144,65536,16.467234293619793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,4096,0.5927182303534614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,6144,1.623202641805013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,4096,0.9563297695583768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,3072,0.5129262076483833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,3584,0.6178133222791884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,5120,1.3681572808159723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,2048,0.35626310772365993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,3072,0.7064595752292209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,3584,0.8919341829088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,2560,0.38828354411655003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,1536,0.2871626747979058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,2560,0.6585520108540853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,2048,0.5366471078660753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,1024,0.17663289441002739
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,768,0.15291466977861193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,512,0.10646666420830621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,256,0.0658515559302436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,1024,0.41199644406636554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,1536,0.468032889895969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,768,0.3650515609317356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,128,0.037244445747799344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,64,0.027648889356189307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,32,0.02181688944498698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,512,0.327078209982978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,256,0.30049777030944824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,128,0.2561182181040446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,64,0.2595591015285916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,32,0.2493600050608317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,12288,1.6525999704996746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,16384,2.0523484547932944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,5120,65536,11.5626949734158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,10240,1.3044266170925565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,8192,1.423365381028917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,16384,3.753704918755425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,10240,2.36317867702908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,12288,2.828467475043403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,7168,0.9153866238064237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,8192,1.9099085066053603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,6144,0.6795048713684082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,5120,0.620231999291314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,7168,1.6513982348971898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,4096,0.5451191266377767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,3584,0.46997955110337997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,6144,1.4033786985609267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,5120,65536,15.637045966254341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,4096,0.7883466614617242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,5120,1.1532942454020183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,3072,0.4425448841518826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,3584,0.6671679814656576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,2560,0.3194399939643012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,3072,0.5132568677266439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,2048,0.2508782280815972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,2048,0.37774311171637637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,2560,0.42520711157057023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,1536,0.21706044673919678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,1024,0.13332266277737087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,512,0.07892355653974745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,768,0.10837777455647786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,1536,0.3273671203189426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,1024,0.22053955660925972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,512,0.15028977394104004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,128,0.026957333087921143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,256,0.04430133435461256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,768,0.18323555257585314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,64,0.02219733264711168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,32,0.01737600068251292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,256,0.12389332718319362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,128,0.11028977897432114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,64,0.10472355286280315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,32,0.09728266795476277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,4096,65536,9.074024624294704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,12288,1.4505315356784398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,16384,1.9465217590332031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,10240,1.569783952501085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,12288,2.7441671159532337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,8192,1.1292791366577148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,16384,3.6401341756184897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,10240,2.2870267232259116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,7168,0.8692257669236926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,6144,0.7842853334214953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,7168,1.6089466942681208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,5120,0.6619964705573188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,8192,1.8376390669080946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,6144,1.3590435451931422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,4096,0.5422613355848525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,3584,0.4837510850694444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,4096,0.7283706665039062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,5120,1.1422017415364583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,3072,0.41883465978834367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,3584,0.6033191151089138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,2560,0.31146666738722056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,2048,0.2841511037614611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,3072,0.5135475794474283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,1536,0.22561600473192003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,2048,0.3127173317803277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,1024,0.14043377505408391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,1536,0.2849244541592068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,768,0.08997511201434666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,2560,0.3904213375515408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,1024,0.21257689264085558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,512,0.07687911060121325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,768,0.16678667068481445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,256,0.036997334824668035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,128,0.02289688918325636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,512,0.13311644395192465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,64,0.020382222202089097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,32,0.016632889707883198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,256,0.11265688472323948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,128,0.09869422515233357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,64,0.08909333414501613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,4096,65536,14.887507120768229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3584,65536,7.470335218641494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,32,0.09103466404808892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,10240,0.9265404807196723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,12288,1.3284240298800998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,16384,1.6902799606323242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,8192,0.9177964528401693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,12288,2.754938761393229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,10240,2.204837375217014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,16384,3.5204986996120877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,7168,0.6403191354539659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,6144,0.5441217952304417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,8192,1.7596142027113173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,5120,0.4331013361612956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,4096,0.38001243273417157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,3584,0.29092444313897026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,7168,1.5356008741590712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,6144,1.3160666359795463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,4096,0.6901555591159396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,5120,1.0774355994330513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,3072,0.2657208972507053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,2560,0.2268995443979899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,2048,0.20036178165011934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,3584,0.5814062224494086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,3072,0.45068269305759007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,2560,0.36114221149020725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,1536,0.15975822342766657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,65536,6.153365241156684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,1024,0.09577155775494045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,2048,0.2924648920694987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,1536,0.22738666004604766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,512,0.06276800235112508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,768,0.08372533321380615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,1024,0.1814675596025255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,256,0.03179644544919332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,128,0.023374223046832617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,64,0.01889333294497596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,768,0.16665777895185682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,512,0.11751643816630046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,256,0.09453066852357651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,3072,32,0.013935999737845527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,128,0.0856364435619778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,64,0.08755555417802598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,32,0.08191288842095269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3584,65536,14.390206231011284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,12288,0.8676888677808973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,16384,1.272414207458496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,8192,0.6755626466539172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,10240,0.7875048849317762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,7168,0.7513626416524252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,12288,2.561189227634006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,10240,2.122671127319336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,16384,3.3860382503933377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,6144,0.5330906444125706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,8192,1.7087120480007594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,5120,0.41520176993476016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,7168,1.4930986828274195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,4096,0.39141154289245605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,6144,1.2535822126600478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,3584,0.3497920036315918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,65536,4.873438093397352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,4096,0.6443475617302789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,5120,1.0268835491604276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,3584,0.5402257707383897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,2560,0.2489466667175293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,2048,0.20507733027140299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,3072,0.2943093246883816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,2560,0.33518844180636936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,1536,0.16597421964009604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,3072,0.42772356669108075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,1024,0.09276266892751057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,2048,0.2675795555114746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,768,0.07781066497166951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,1536,0.20012711154090035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,512,0.0529404448138343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,128,0.018262222409248352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,1024,0.15856888559129503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,768,0.12605422072940403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,256,0.02833777666091919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,512,0.10040799776713054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,64,0.014974221587181091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,256,0.08206844329833984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2560,32,0.013268444273206921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,128,0.07715911335415311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,32,0.06598844130833943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,64,0.06877422332763672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,12288,0.7297209103902181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,16384,1.0207448535495334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,10240,0.6609093348185221
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,3072,65536,13.951537238226997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,8192,0.5609022246466743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,7168,0.469271977742513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,12288,2.4605316585964627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,10240,2.0499511294894748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,6144,0.3666391107771132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,16384,3.2690355512830944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,5120,0.32344889640808105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,8192,1.660192913479275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,4096,0.25231554773118764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,65536,4.087301466200087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,3584,0.2223280005984836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,3072,0.19415111011928984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,7168,1.4116684595743816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,6144,1.1991280449761283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,4096,0.5894373257954916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,5120,0.9903715981377496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,2560,0.14202133814493814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,3584,0.543357319302029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,3072,0.39481067657470703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,2048,0.13356799549526638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,1024,0.06790400213665433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,1536,0.0960728857252333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,768,0.06784177488750882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,2048,0.23791021770901152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,1536,0.18027822176615396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,2560,0.30839022000630695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,512,0.041179554329978094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,1024,0.13680622312757704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,256,0.021716444028748408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,768,0.11317243840959336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,512,0.08462488651275635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,256,0.06910222106509738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,64,0.012060444388124677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,128,0.06072000000211927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,32,0.009924444059530893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,2048,128,0.015050666199790107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,64,0.057164443863762744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,32,0.056206219726138644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,12288,0.6021973292032877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,16384,0.8967190848456489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,10240,0.616063117980957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,8192,0.43413954310946995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2560,65536,13.402089436848959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,7168,0.4191271199120416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,65536,3.500374264187283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,12288,2.366582234700521
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,16384,3.139987521701389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,10240,1.9792630937364366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,6144,0.41257066196865505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,5120,0.28306576940748424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,8192,1.5653004116482203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,4096,0.2634791003333198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,3584,0.20427466763390434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,4096,0.5671315722995335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,6144,1.1575519773695204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,7168,1.372470219930013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,3072,0.17698221736484102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,3584,0.4799777666727702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,5120,0.9324791166517469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,2560,0.13221422831217447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,2048,0.1086168885231018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,3072,0.362503104739719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,1536,0.08515199687745835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,1024,0.06450221935908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,768,0.04404533240530226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,1536,0.1767439974678887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,2048,0.23736445109049478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,512,0.030489779180950586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,2560,0.28755733701917857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,256,0.01788444485929277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,128,0.012246222131782107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,1024,0.1004213359620836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,768,0.08701955609851414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,64,0.01000533335738712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1536,32,0.008802666432327693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,256,0.05639288822809855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,512,0.06918044222725762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,128,0.049181332190831505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,64,0.045372443066702954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,32,0.0440684457619985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,12288,0.36658133400811094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,16384,0.4958648681640625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,10240,0.3170551194085015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,8192,0.2759297688802083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,65536,2.0059191385904946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,7168,0.22756266593933105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,2048,65536,12.8264651828342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,12288,2.266364415486654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,10240,1.8923288981119792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,6144,0.17124177349938285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,5120,0.1460311147901747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,8192,1.5058221817016602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,4096,0.11445511711968316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,3584,0.10523911317189534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,16384,3.0239234500461154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,3072,0.09158577521642049
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,3584,0.44069867663913304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,4096,0.5246284272935655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,2560,0.07869955566194323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,7168,1.3114248911539714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,6144,1.1009093390570748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,2048,0.06762399938371447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,5120,0.9055289162529839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,3072,0.3316168785095215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,1536,0.04557866520351834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,1024,0.03562577896647983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,768,0.029049777322345312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,2560,0.2560346656375461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,512,0.019904888338512845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,1536,0.14837155077192518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,256,0.012787555654843649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,1024,0.08111111323038737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,2048,0.20823555522494844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,128,0.008466666771305932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,768,0.0658151110013326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,512,0.05383466680844625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,64,0.006884444504976273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,1024,32,0.005972444597217772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,256,0.04366933306058248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,128,0.03653066688113742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,64,0.033428443802727595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,32,0.032680888970692955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,12288,0.2776248984866672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,16384,0.40255554517110187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,10240,0.294906669192844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,65536,1.922920862833659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,8192,0.2756933371225993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,7168,0.18639732731713188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,10240,1.900296105278863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,12288,2.270387649536133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1536,65536,12.309044731987846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,6144,0.17554222212897408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,16384,3.0140516493055554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,5120,0.15696977244483099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,4096,0.12911289268069798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,3584,0.11611110634273952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,8192,1.5105928844875758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,3072,0.10215910938051011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,5120,0.89828003777398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,7168,1.2929599550035265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,3584,0.4403982162475586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,4096,0.5662417941623265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,2560,0.0936080018679301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,6144,1.097794638739692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,2048,0.06828533278571235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,1536,0.049738665421803795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,3072,0.334208885828654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,1024,0.034113778008355036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,768,0.027407111393080816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,2048,0.2027653323279487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,2560,0.2554355462392171
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,1024,0.07553422451019287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,768,0.06064889166090223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,1536,0.13925245073106554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,512,0.019325332509146798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,256,0.012034667034943899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,128,0.008099555969238281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,64,0.006276444428496891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,768,32,0.005311111195219888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,512,0.04981155527962578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,128,0.03381155596839057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,64,0.030641777647866145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,32,0.029548443026012842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,256,0.03961955507596334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,16384,0.2631431155734592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,12288,0.19876089360978869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,10240,0.16246488359239367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,8192,0.14537244372897679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,65536,1.5235431459214952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,7168,0.1806559960047404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,8192,1.4301012886895075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,10240,1.8086649576822917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,6144,0.08735555410385132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,12288,2.1744683583577475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,5120,0.07712355587217543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,4096,0.06272711356480916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,7168,1.248011589050293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,3584,0.0563653310139974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,1024,65536,11.874405754937065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,16384,2.904816097683377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,4096,0.4895920223659939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,3072,0.0503022207154168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,2560,0.03924444317817688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,2048,0.04077688852945963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,3584,0.41129689746432835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,5120,0.8427617814805773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,6144,1.054568926493327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,1536,0.028328001499176025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,3072,0.3136417865753174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,1024,0.01882844501071506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,768,0.015531554818153381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,2560,0.24479111035664877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,1536,0.11354755030737983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,1024,0.06782133049435086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,2048,0.1784435510635376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,512,0.011319999893506369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,256,0.007393777370452881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,768,0.047600001096725464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,128,0.005520888914664586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,64,0.004580444345871608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,512,32,0.003987555702527364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,128,0.024110222856203716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,512,0.03754666778776381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,256,0.02915289004643758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,64,0.021949332621362474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,32,0.020948444803555805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,12288,0.19094488355848524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,16384,0.2502302328745524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,10240,0.08523644341362847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,8192,0.06825688812467787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,65536,0.8848720126681857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,7168,0.06079555882347954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,8192,1.409057829115126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,10240,1.7872861226399739
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,6144,0.05284533235761854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,12288,2.1551261478000217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,5120,0.044616887966791786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,7168,1.2247449027167427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,4096,0.03734666771358914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,16384,2.8797181447347007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,4096,0.4827129046122233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,5120,0.8381848865085177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,3584,0.033342222372690834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,768,65536,11.859056260850695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,3072,0.02962133288383484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,6144,1.030264006720649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,2048,0.021793777743975323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,2560,0.028204444381925795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,3584,0.388504876030816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,1536,0.01835466590192583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,2560,0.1942897770139906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,3072,0.2840800020429823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,2048,0.13026400407155356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,1024,0.012453333371215396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,768,0.01075466639465756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,1024,0.04961599906285604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,1536,0.07577244440714519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,512,0.008176888856622908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,512,0.03286755416128371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,256,0.0058471109304163195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,768,0.04095200035307143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,128,0.004600000050332812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,64,0.0036675557494163513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,256,32,0.0035662220584021676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,256,0.02488533324665493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,128,0.02013688948419359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,64,0.017629333668284945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,32,0.016902221573723685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,12288,0.12291022141774495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,16384,0.22631822692023384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,10240,0.043271111117468945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,65536,0.8483609093560113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,8192,0.035122666094038225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,7168,0.03123733401298523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,10240,1.7901191711425781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,7168,1.209302266438802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,12288,2.1437839931911893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,6144,0.028049776951471966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,8192,1.4046684900919597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,5120,0.023856000767813787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,4096,0.024491555160946313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,3584,0.023544889357354905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,16384,2.876189337836372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,3072,0.020983111527231004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,4096,0.4768968688117133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,3584,0.37372178501553005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,2560,0.019256888164414298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,2048,0.01499111122555203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,5120,0.8340106540256076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,3072,0.26224533716837567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,512,65536,11.438868204752604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,2560,0.17027110523647734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,1536,0.012882666455374824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,6144,1.0210498174031575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,2048,0.10256266593933105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,768,0.0075235557225015424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,512,0.005993777679072485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,1024,0.008982222113344405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,1536,0.06459822257359822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,1024,0.04725777771737841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,256,0.004511111312442356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,512,0.030656887425316706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,768,0.038930667771233454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,64,0.0029688889367712867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,128,0.003710222327046924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,128,32,0.0029022221763928733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,256,0.022616000639067754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,128,0.018262222409248352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,64,0.015746666325463187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,32,0.014864888456132678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,12288,0.09568711121877034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,10240,0.03992088966899448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,16384,0.24158755938212076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,8192,0.032989333073298134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,65536,0.8284702301025391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,7168,0.048955556419160634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,8192,1.401256031460232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,6144,0.04150399896833632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,10240,1.7796897888183594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,7168,1.2084621853298612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,12288,2.149817784627279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,5120,0.03678133421474033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,4096,0.027600000301996868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,3584,0.025938666529125635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,4096,0.4864071210225423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,3584,0.3724035686916775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,3072,0.022074666288163926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,16384,2.871538586086697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,2560,0.01959288948112064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,2048,0.014931556251313953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,5120,0.8276586532592773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,1536,0.012493333054913415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,6144,1.0209787156846788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,3072,0.24675112300448945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,1024,0.008797333472304875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,2048,0.10001155402925278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,768,0.00741155528359943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,1536,0.06389777527915107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,2560,0.16660088962978786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,512,0.005668444352017508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,1024,0.046816888782713145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,256,0.0041457778877682155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,256,65536,11.399576822916666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,768,0.03873777720663283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,128,0.0033973333322339584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,64,0.002522666628162066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,512,0.03040177623430888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,256,0.02221244408024682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,64,32,0.002354666590690613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,64,0.015695111619101632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,32,0.014862222803963555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,128,0.018301332990328472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,16384,0.21600533856285942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,12288,0.062259554862976074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,10240,0.038966221941841975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,65536,0.8176906903584799
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,8192,0.032207998964521624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,7168,0.041464000940322876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,8192,1.4026017718844945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,10240,1.7754862043592665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,12288,2.1484177907307944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,6144,0.03221866819593642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,5120,0.028351998991436426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,4096,0.021202666891945735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,16384,2.8696373833550344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,4096,0.4650106430053711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,3584,0.021596444977654353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,3072,0.0185199992524253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,5120,0.8273182445102267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,7168,1.2152302000257704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,2560,0.015112888481881885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,2048,0.01149866647190518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,6144,1.021864891052246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,3584,0.36808710628085667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,3072,0.25320622656080455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,1536,0.009826666778988307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,2560,0.16720710860358345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,2048,0.10019910997814602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,1024,0.0069919998447100324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,768,0.005935110979610019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,512,0.00442844463719262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,128,65536,11.372654385036894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,1536,0.0636319981680976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,256,0.003674666500753827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,1024,0.046727110942204796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,768,0.03845955597029792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,128,0.0028080000645584534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,64,0.002463111033042272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,512,0.030219554901123047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4096,32,32,0.0023599999646345773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,128,0.018139556050300598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,64,0.015746666325463187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,32,0.01479022204875946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,256,0.022304000125990972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,64,65536,11.414859347873263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,10240,5.73265881008572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,12288,6.8347218831380205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4096,32,65536,11.375518798828125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,16384,9.041915893554688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,10240,10.219545152452257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,8192,4.566114637586805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,7168,4.15260272555881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,8192,8.13742913140191
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,12288,12.234489440917969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,6144,3.468234592013889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,7168,7.100919935438369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,5120,2.9817725287543406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,4096,2.4708008236355252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,6144,6.159943050808376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,5120,5.012999216715495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,4096,4.133242713080512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,16384,16.25530836317274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,3584,2.234346601698134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,3072,2.0176995595296225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,3584,3.546659469604492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,2560,2.579782274034288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,2560,1.7414151297675238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,2048,1.5181289248996312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,3072,3.052872975667318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,2048,2.0381768544514975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,1536,1.6037297778659398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,512,0.7651911311679416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,768,0.968807962205675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,256,0.5070337719387478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,1536,1.278182241651747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,768,0.9692889319525825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,1024,1.0669742160373263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,512,0.8795608944363065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,1024,1.092026710510254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,64,0.46474843555026585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,128,0.5166151258680556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,65536,32,0.595686223771837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,64,0.6810702217949761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,32,0.66248533460829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,128,0.7597857581244575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,65536,256,0.7898337576124402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,10240,2.5663324991861978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,12288,2.405000898573134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,12288,3.107817755805122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,16384,3.1827386220296225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,16384,4.114760080973308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,8192,1.4586675431993272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,8192,2.0341964297824435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,7168,1.2700497309366863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,10240,2.063751008775499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,7168,1.7934417724609375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,6144,1.5122897889879015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,5120,0.909953753153483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,6144,1.0639368693033855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,4096,0.964943144056532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,5120,1.2513244416978624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,4096,0.7772906621297201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,3584,0.8777670860290527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,3584,0.6733804278903537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,3072,0.7052222357855903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,3072,0.6345902019076878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,2560,0.5543724695841471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,2560,0.5902141994900173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,1536,0.40154133902655703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,2048,0.5483137766520182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,1024,0.28413155343797475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,1024,0.3292284541659885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,2048,0.4551982349819607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,1536,0.3899119959937201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,768,0.2332515451643202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,256,0.09376444419225057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,512,0.1776977777481079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,768,0.30851999918619794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,64,0.04414133230845133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,256,0.20997511015997994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,128,0.06180622180302938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,32,0.032285332679748535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,512,0.2459368970659044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,128,0.19570933447943795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,64,0.18359911441802979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,32,0.17889777819315592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,12288,2.0192898644341364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,12288,2.324321746826172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,16384,2.6908274756537547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,16384,3.1030125088161893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,10240,1.9435306125217016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,10240,1.676215065850152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,8192,1.211350229051378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,8192,1.5111671023898656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,16384,65536,12.875675625271269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,6144,0.8732417954338921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,7168,1.0194133122762044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,6144,1.092942237854004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,7168,1.3137075636121962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,5120,0.9724186791314019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,3584,0.6682480176289877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,4096,0.6736702389187283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,3584,0.5641342269049751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,4096,0.6251351038614908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,5120,0.7427831225925021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,3072,0.5162711143493652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,2560,0.4231875472598606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,2048,0.3653564453125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,3072,0.5177306599087185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,16384,65536,16.066560533311634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,2560,0.4421351220872667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,1536,0.26702221234639484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,1024,0.20939377943674722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,2048,0.3698648876614041
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,768,0.16213244862026638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,512,0.11833600203196208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,1536,0.3107057677374946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,1024,0.27145687739054364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,256,0.06182666619618734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,768,0.21933866871727836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,64,0.033879998657438494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,512,0.19013244575924346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,128,0.04231289029121399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,256,0.17378666665818956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,64,0.13976532883114284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,32,0.025054223007626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,32,0.13727466265360513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,128,0.1554364495807224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,12288,1.917234632703993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,12288,2.0167734358045792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,16384,2.58168093363444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,16384,2.579941431681315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,12288,65536,10.861035664876303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,12288,65536,11.921901279025606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,10240,1.5644355350070531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,8192,1.2222791247897677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,8192,1.0732417636447482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,7168,1.0873289108276367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,10240,1.6088373396131728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,7168,0.9736187193128797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,6144,0.7877804438273112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,6144,0.9338062074449328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,4096,0.6361111005147299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,5120,0.7451955477396647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,5120,0.6766417821248373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,2560,0.3610951105753581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,3072,0.4569990899827745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,3072,0.5664026472303603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,3584,0.5638995700412327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,4096,0.5462817615932888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,3584,0.4813689125908746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,2048,0.33126311832004124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,1536,0.22231645054287383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,2560,0.3809831142425537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,1536,0.2663866678873698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,2048,0.3337093459235297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,1024,0.1867306629816691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,1024,0.21220355563693574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,768,0.12321244345770942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,512,0.10287821955151027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,768,0.1822497712241279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,128,0.030807110998365615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,256,0.06357510884602864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,512,0.16730577415890166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,64,0.026722666290071275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,256,0.13616266515519884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,32,0.022279999322361414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,128,0.12567467159695095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,64,0.11877866586049397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,32,0.1177004443274604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,12288,1.581308470831977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,16384,2.0528808169894748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,16384,2.448848088582357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,10240,1.2643306520250108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,10240,65536,9.887705485026041
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,12288,1.838255140516493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,10240,1.528114636739095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,10240,65536,10.32501475016276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,8192,0.9368569056193033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,7168,0.9218720330132378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,8192,1.0731955634223092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,5120,0.629676447974311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,7168,0.9494436052110461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,6144,0.8001911375257703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,6144,0.6551591025458442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,4096,0.5129297574361166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,5120,0.7061982154846191
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,3584,0.3862960073682997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,4096,0.5761626561482748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,3072,0.38543910450405544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,3072,0.5154933399624294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,2048,0.24503554238213432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,2560,0.34094844924079043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,2560,0.4512622091505263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,2048,0.411314672893948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,1536,0.1852382156583998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,3584,0.558018684387207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,1024,0.17218489117092559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,768,0.10734666718377008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,512,0.07208622164196439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,256,0.043805334303114146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,768,0.26846933364868164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,128,0.025255999631351892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,1536,0.35443732473585343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,64,0.022441777918073866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,1024,0.2926151222652859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,32,0.017311111092567444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,512,0.24633688396877715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,256,0.22165421644846597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,128,0.2025973399480184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,64,0.20755555894639757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,32,0.19719378153483072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,8192,65536,7.866341484917535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,12288,1.3552719751993816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,16384,1.7934239705403645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,12288,1.8038177490234375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,10240,1.1287493175930448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,16384,2.4048213958740234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,8192,1.0374293857150607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,8192,65536,9.172682868109808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,10240,1.523191981845432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,8192,1.048357327779134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,7168,0.8294249110751681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,7168,0.9354453616672092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,6144,0.7201075553894043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,5120,0.5799599753485786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,6144,0.7850115564134387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,5120,0.6638951301574707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,3584,0.4556328985426161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,4096,0.5216524336073134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,4096,0.5465893215603298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,3072,0.40169866879781085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,3584,0.4954204559326172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,2048,0.2787982092963325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,1536,0.1864373286565145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,2560,0.3144746621449788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,3072,0.46944888432820636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,2560,0.4389199945661757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,2048,0.36956267886691624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,1024,0.1257422235276964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,768,0.10229955779181586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,512,0.07786577939987183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,256,0.03680800067053901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,1024,0.28314577208624947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,128,0.02260355485810174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,1536,0.34166667196485734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,512,0.22961333062913683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,64,0.02011999984582265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,256,0.19754310448964438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,32,0.016167110866970487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,768,0.2489795684814453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,128,0.19094133377075195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,64,0.18533244397905138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,32,0.17981155713399252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,12288,1.1060809029473198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,7168,65536,6.94693586561415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,16384,1.5629084904988606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,16384,2.1572337680392795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,10240,1.07752619849311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,10240,1.3595057593451605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,12288,1.624683592054579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,8192,0.6737333403693305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,7168,0.5909271240234375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,8192,0.9553012847900391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,6144,0.5496764183044434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,7168,0.8101742002699109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,7168,65536,9.124944898817274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,5120,0.4939582082960341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,6144,0.6971679793463813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,4096,0.3832542101542155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,3584,0.3420071072048611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,5120,0.5924791230095757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,3072,0.26491822136773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,4096,0.46800533930460614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,2560,0.22529421912299263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,2048,0.188481781217787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,3584,0.4282977845933702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,3072,0.39536534415351016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,1536,0.15187466144561768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,1024,0.10065778096516927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,2560,0.37127288182576496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,768,0.08763288789325291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,2048,0.32177066802978516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,1024,0.2334364520178901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,1536,0.2940453423394097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,256,0.03503200080659654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,512,0.05574488639831543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,768,0.2108151117960612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,128,0.02121155626244015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,32,0.014341332846217685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,64,0.01847199930085076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,512,0.181931561893887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,256,0.15675466590457493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,128,0.1509608825047811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,64,0.14734133084615073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,32,0.15450578265719944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,6144,65536,6.2015838623046875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,12288,0.8814169036017524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,16384,1.2096213234795463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,10240,0.811757352617052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,8192,0.6544631322224935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,12288,1.5808720058865016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,16384,2.1216676500108504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,10240,1.317024866739909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,7168,0.6560177803039551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,8192,0.9446026484171549
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,6144,0.5605937639872233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,6144,65536,8.22120836046007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,7168,0.7938586870829264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,5120,0.45897865295410156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,4096,0.37436887953016496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,6144,0.6494631237453884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,3584,0.3289324442545573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,5120,0.5358053313361274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,3584,0.39809245533413357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,3072,0.3040657838185628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,4096,0.4510142008463542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,2560,0.23553066783481172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,3072,0.3559306727515326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,2560,0.3127288818359375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,1536,0.14059910509321424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,1024,0.09455911318461101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,2048,0.19059999783833823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,2048,0.3051662180158827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,512,0.04758044415050083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,768,0.07688533597522311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,1536,0.2602870994144016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,256,0.027277333868874445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,128,0.017695110705163743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,768,0.18445066610972086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,64,0.01534400052494473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,512,0.147361781862047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,1024,0.20633244514465332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,256,0.13231821854909262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,32,0.0125946667459276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,64,0.12635200553470188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,32,0.12488444646199544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,128,0.13234221935272217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,5120,65536,5.126378801133898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,12288,0.7380995750427246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,16384,1.0255724589029949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,10240,0.7206933233473035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,8192,0.524895085228814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,12288,1.3741849263509114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,16384,1.8478605482313368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,7168,0.42152799500359434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,10240,1.1369235780504015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,8192,0.7833609051174588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,6144,0.3554684321085612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,7168,0.6142960124545628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,5120,0.3134720060560438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,6144,0.4900302357143826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,4096,0.2270809014638265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,5120,0.3981777826944987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,3584,0.21949156125386557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,4096,0.315299563937717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,3584,0.2645182344648573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,3072,0.1781484418445163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,2560,0.14894489447275797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,3072,0.2401475641462538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,2048,0.1179928912056817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,1536,0.09874755806393093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,2048,0.15254310766855875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,2560,0.18591288725535074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,1024,0.06524799929724799
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,768,0.04928000105751885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,1536,0.13335466384887695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,5120,65536,8.195799933539496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,1024,0.09886222415500218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,512,0.035931554105546736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,128,0.014281777871979607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,768,0.0921635561519199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,256,0.022448889083332483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,65536,4.082125345865886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,64,0.013136000268989138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,512,0.07748355468114217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,256,0.06649066342247857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,4096,32,0.01166044423977534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,128,0.05853688716888428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,32,0.055461333857642285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,64,0.05563466416464912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,12288,0.7036426862080892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,16384,0.7999537785847982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,10240,0.5582364400227865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,8192,0.4902702437506781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,12288,1.3577138053046331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,7168,0.4125688870747884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,16384,1.8427715301513672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,10240,1.1296595467461479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,8192,0.7144435776604546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,7168,0.6052684254116482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,6144,0.3590879970126682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,5120,0.30579643779330784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,6144,0.4764889081319173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,3584,0.21997066338857016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,4096,0.24081601036919487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,5120,0.3822160032060411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,2560,0.1481226682662964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,3072,0.24540177981058756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,3072,0.18703822294871011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,3584,0.259805334938897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,4096,0.2992782327863905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,2048,0.11638311545054118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,2560,0.1986044380399916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,1536,0.09364444679684109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,2048,0.14886755413479275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,1024,0.07104000118043688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,768,0.05019911130269369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,1536,0.12935645050472683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,512,0.03840622305870056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,256,0.023172444767422144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,1024,0.09325510925716823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,65536,3.5578443739149304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,768,0.08498844173219468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,4096,65536,7.458858066134983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,128,0.014919110470347934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,512,0.0723537802696228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,256,0.06185511085722181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,64,0.013565333353148567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,128,0.0547964440451728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3584,32,0.0094373333785269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,64,0.052248888545566134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,32,0.05309155583381653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,10240,0.43444710307651097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,12288,0.5778560108608669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,16384,0.7562693489922417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,12288,1.2713395224677193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,8192,0.46356889936659074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,8192,0.6960524453057183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,10240,1.0498089260525174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,7168,0.42684443791707355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,16384,1.7201467090182836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,7168,0.574170642428928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,5120,0.3126800060272217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,6144,0.3425893253750271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,5120,0.34679555892944336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,6144,0.4303555488586426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,3584,0.2052853372361925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,4096,0.2456746631198459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,4096,0.28986933496263295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,3072,0.1757591168085734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,2560,0.14848889244927302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,3584,0.24986312124464247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,3072,0.21370489067501494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,2048,0.09511733055114746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,1536,0.08464888731638591
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,2560,0.1786053313149346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,2048,0.14775999387105307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,1536,0.09780355294545491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,1024,0.0797226693895128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,768,0.042845331960254245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,65536,3.484361860487196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,512,0.030215998490651447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,256,0.022444445225927565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,768,0.07157155540254381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,128,0.015240000353919135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,512,0.06443555487526788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,1024,0.06779911120732625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,256,0.053373333480623036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,32,0.008354666332403818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,64,0.04473511046833462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,3072,64,0.011512888802422417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,128,0.04739377895991007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,32,0.04389155573315091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3584,65536,7.455238342285156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,12288,0.5358995331658257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,16384,0.6028960016038682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,10240,0.4623306592305501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,8192,0.3802808920542399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,12288,1.2652506298489041
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,16384,1.720621320936415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,10240,1.0556844075520833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,7168,0.28042401207817924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,6144,0.2589457829793294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,8192,0.6499066882663304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,5120,0.21216000450981987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,4096,0.20595199531979033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,6144,0.42600266138712567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,65536,2.6820977528889975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,7168,0.5519350899590386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,3584,0.18760622872246635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,4096,0.28296444151136607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,5120,0.3424817721048991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,3072,0.1614097754160563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,2560,0.14719288878970677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,3072,0.20361955960591635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,2048,0.11098488834169175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,3584,0.24034044477674696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,1024,0.06530133220884535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,2560,0.16660355197058782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,1536,0.1020595563782586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,768,0.0413626664214664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,512,0.03383022215631273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,1024,0.07532711161507501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,256,0.02057422200838725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,2048,0.12929688559638128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,768,0.06807021962271796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,1536,0.09367911020914714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,128,0.0138586668504609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,64,0.009936889012654623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2560,32,0.00962044464217292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,256,0.04948444498909844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,64,0.041342222028308444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,32,0.0407608879937066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,128,0.045928001403808594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,3072,65536,6.961654239230686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,512,0.05896711349487305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,10240,0.2975173261430529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,12288,0.33547022607591415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,16384,0.47989866468641496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,8192,0.24461865425109863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,7168,0.2060453361935086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,8192,0.6147395769755045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,6144,0.1750088930130005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,12288,1.193687121073405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,10240,0.9680532879299588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,7168,0.49497244093153214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,5120,0.1495786640379164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,16384,1.5922595130072699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,6144,0.38930667771233457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,4096,0.11656978395250107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,3584,0.10205511252085368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,65536,2.200488832261827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,3072,0.08822755681143866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,5120,0.3030737770928277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,4096,0.24195377031962076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,2560,0.09057244327333237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,2048,0.05856533182991875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,3584,0.20807733800676134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,3072,0.17293243938022188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,2048,0.10753599802652995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,1024,0.038167999850379095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,2560,0.13949067062801784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,1536,0.051386667622460254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,768,0.029466665453381006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,512,0.020012444920010038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,1536,0.076072891553243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,1024,0.06070666842990451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,256,0.012279999752839407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,768,0.05413688884841072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,128,0.008434666527642144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,64,0.006735111276308696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,2048,32,0.005973333285914526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,256,0.03985866573121812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,512,0.04982044299443563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,128,0.03499822152985467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,32,0.03197955422931247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,64,0.032638221979141235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,12288,0.2897084554036458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,16384,0.3949395550621881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,10240,0.307861328125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2560,65536,6.916614108615452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,65536,1.4058373769124348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,8192,0.2060826619466146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,7168,0.20523199770185682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,12288,1.1827590730455186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,16384,1.6039937337239583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,10240,0.9748737547132703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,6144,0.16497688823276097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,8192,0.5898826917012533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,5120,0.14089866479237875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,4096,0.13358488347795275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,6144,0.3786826663547092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,5120,0.29776978492736816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,7168,0.4904799991183811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,3584,0.11867022514343262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,4096,0.23344887627495658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,3072,0.10150844520992702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,2560,0.09039555655585395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,2048,0.07884355386098225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,1536,0.050055109792285495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,3072,0.1641626622941759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,1024,0.034734222624037005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,2048,0.09134488635592991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,2560,0.13388800621032715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,1536,0.07049422131644355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,768,0.02786311176088121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,256,0.011794666449228922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,1024,0.05711466736263699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,512,0.019853333632151287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,3584,0.20628710587819418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,768,0.05038488904635111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,128,0.00813333359029558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,64,0.006249777972698212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,512,0.04319733381271362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1536,32,0.0053119998839166425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,256,0.03604000144534641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,128,0.03200088938077291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,64,0.029678222205903795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,32,0.028947555356555518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,16384,0.24803466267055937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,12288,0.2005164490805732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,10240,0.161263108253479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,8192,0.13016622596316868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,2048,65536,6.411814371744792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,65536,1.1608080334133573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,7168,0.10596978002124363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,10240,0.8848693105909559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,12288,1.087810622321235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,8192,0.5271031061808268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,6144,0.09228266610039605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,5120,0.07449955410427518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,4096,0.060849779182010226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,3584,0.05375111103057861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,7168,0.43779555956522626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,16384,1.4845964643690321
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,6144,0.33279021581013996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,5120,0.24301245477464464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,3072,0.04948355423079597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,4096,0.18800000349680582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,2560,0.03962577713860406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,1536,0.02872977654139201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,3584,0.1545191076066759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,2048,0.03387733300526937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,1024,0.019289778338538278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,3072,0.12355200449625652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,2560,0.09830400016572739
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,2048,0.06560355424880981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,768,0.015372445185979208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,1536,0.053223109907574125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,512,0.011151110960377587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,256,0.007359111474619971
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,128,0.005570666657553778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,1024,0.04272355635960897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,64,0.0046302220887608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,512,0.031172444423039753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,768,0.03668177790111966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,1024,32,0.003992889076471329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,256,0.025966222087542217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,128,0.02254044512907664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,64,0.02104622291194068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,32,0.02017244365480211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,16384,0.21104799376593697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,12288,0.15792266527811685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,10240,0.13788621955447727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,8192,0.1158648861779107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,65536,0.8410151269700793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,7168,0.11141955852508545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,8192,0.512664000193278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,10240,0.8818320168389214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,6144,0.08595377869076198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,12288,1.0730426576402452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1536,65536,6.408783806694879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,5120,0.07219200001822577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,4096,0.05819822020000882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,16384,1.471756405300564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,7168,0.41884533564249676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,6144,0.3038080003526476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,4096,0.16427732838524714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,5120,0.22619022263420951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,3584,0.0512586666478051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,3072,0.04528799984190199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,2048,0.0329422222243415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,2560,0.03825689024395413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,1536,0.026366222235891554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,3584,0.13913244671291775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,3072,0.10511910915374756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,2560,0.07791733079486422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,1024,0.018264888061417472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,2048,0.059188445409139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,768,0.014826666977670459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,1536,0.04866400029924181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,512,0.01092711091041565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,1024,0.03830044468243917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,256,0.007147555549939473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,768,0.0336497790283627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,128,0.0053048887186580235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,512,0.02867022156715393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,64,0.004440000073777305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,256,0.023609777291615803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,768,32,0.0037751110891501107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,128,0.0203155560625924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,64,0.01864622202184465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,32,0.018087110585636564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,16384,0.12030666404300266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,12288,0.09220266342163086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,10240,0.07982310983869764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,65536,0.6090702480740017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,8192,0.08890844715966119
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,10240,0.8677031199137369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,8192,0.506040891011556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,12288,1.0672693252563477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,7168,0.060362663533952504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,1024,65536,5.992343054877387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,6144,0.05157688922352261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,16384,1.4627226723564997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,5120,0.044461333089404635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,4096,0.0365573333369361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,3584,0.033357332150141396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,5120,0.21034844716389975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,6144,0.297943115234375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,7168,0.4128524462381999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,3072,0.03341599967744615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,4096,0.14584888352288142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,2560,0.029270221789677937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,3584,0.11418310801188152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,2048,0.02184177769554986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,1536,0.01865866614712609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,3072,0.08774666653739081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,2560,0.06923288769192167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,1024,0.012086222569147745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,768,0.010073777702119615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,512,0.007720000214046902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,2048,0.05610577927695381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,1024,0.036292443672815956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,256,0.005541333307822545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,1536,0.046386665768093534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,768,0.031354665756225586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,128,0.0042275554604000514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,64,0.0034062222888072333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,512,32,0.002990222225586573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,256,0.021319111188252766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,128,0.01828622155719333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,64,0.016585777203241985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,32,0.016234666109085083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,512,0.026672000686327618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,16384,0.07284888956281874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,12288,0.04832266767819723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,10240,0.041321777635150485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,8192,0.035184890031814575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,65536,0.5072399775187174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,7168,0.03224710954560174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,8192,0.5032062000698513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,6144,0.027776888675159876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,10240,0.860538641611735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,7168,0.40155378977457684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,5120,0.02374222212367588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,4096,0.020451555649439495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,12288,1.0652133093939888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,6144,0.26371288299560547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,768,65536,5.929484473334418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,5120,0.18747733698950875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,3584,0.019162666466501024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,4096,0.11906222502390544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,2560,0.01872711049185859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,3072,0.017255999975734286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,2048,0.01601777805222405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,16384,1.455566194322374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,3584,0.09422489007314046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,1536,0.012494222157531314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,3072,0.07663289043638441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,1024,0.00904799997806549
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,768,0.007576888634098901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,2560,0.06432266367806329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,2048,0.054294221931033664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,512,0.005977777971161737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,1024,0.03422844409942627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,1536,0.04423822296990288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,768,0.02942399846182929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,256,0.004454222404294544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,512,0.024233778317769367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,128,0.003727999826272329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,64,0.002959111084540685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,256,0.01940533353222741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,256,32,0.0027297778675953546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,128,0.016407999727461074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,64,0.01501511037349701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,32,0.014512888259357877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,16384,0.035106665558285184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,12288,0.028227554427252874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,10240,0.0291244453854031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,65536,0.45505423016018337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,8192,0.023825777901543513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,8192,0.487502204047309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,7168,0.022043555974960327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,10240,0.8592524528503418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,12288,1.0601547029283311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,6144,0.01994044416480594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,5120,0.017110221915774874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,16384,1.4521555370754664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,512,65536,5.915255228678386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,7168,0.38654221428765195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,5120,0.17671644687652588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,4096,0.014083556003040738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,6144,0.24994044833713105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,3072,0.015276443627145557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,3584,0.01368444412946701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,2560,0.013714666995737286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,2048,0.011985777980751462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,3584,0.08737066719267104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,3072,0.07409422265158759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,4096,0.11926844384935166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,2560,0.06386399931377836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,1536,0.008419555922349295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,2048,0.05376088950369093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,1024,0.0063893331421746155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,768,0.005620444400442972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,512,0.004567111117972268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,1536,0.04401510953903198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,256,0.003657777690225177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,1024,0.033843557039896645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,128,0.0033155555526415506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,768,0.029299557209014893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,512,0.024077332682079736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,64,0.0026568888376156488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,256,0.01923199991385142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,128,32,0.0023395555714766183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,128,0.016362667083740234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,64,0.014928888943460254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,32,0.014360888136757744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,16384,0.03280177712440491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,12288,0.026126222478018865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,10240,0.02325600054528978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,8192,0.0184888889392217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,65536,0.43867911232842344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,7168,0.036652445793151855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,8192,0.4913288752237956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,6144,0.031128889984554712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,7168,0.3838960064782037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,10240,0.8563146591186523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,5120,0.026509332987997267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,4096,0.020820443828900654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,12288,1.0554559495713975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,6144,0.24814134173923066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,3584,0.019476445184813607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,5120,0.18209066655900744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,3072,0.016742222838931613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,4096,0.10588000218073527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,2560,0.014304000470373364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,256,65536,5.891660478379991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,16384,1.4525831010606554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,2048,0.011573333707120685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,3584,0.08480799860424465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,3072,0.07384977738062541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,1536,0.009539555344316695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,1024,0.006816888848940532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,2560,0.06378400325775146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,768,0.00572622236278322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,1536,0.043807999955283276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,512,0.00443466669983334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,1024,0.033696889877319336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,768,0.028993778758578833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,256,0.003305777700410949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,2048,0.05380355649524265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,512,0.023952000670962866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,128,0.002716444432735443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,64,0.002258666687541538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,64,32,0.0021191111041439907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,256,0.019104889697498746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,128,0.016214221715927124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,16384,0.0322675539387597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,32,0.014346667461925082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,64,0.014927999840842353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,12288,0.026691555976867676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,10240,0.022453332940737408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,8192,0.01754666699303521
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,65536,0.3856497870551215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,8192,0.49611912833319766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,7168,0.021190222766664293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,10240,0.8567964235941569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,6144,0.01832266648610433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,12288,1.0562622282240126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,5120,0.013284444808959961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,7168,0.3849528895484076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,6144,0.24448977576361763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,16384,1.4539857440524633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,5120,0.17995822429656982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,4096,0.011026666396194033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,128,65536,5.897711859809028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,3072,0.009481777747472128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,3584,0.01053244454993142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,2560,0.008630221916569604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,2048,0.007516444557242923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,4096,0.10511110888587104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,3584,0.08557155397203232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,2560,0.06365955538219877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,1536,0.008921778036488427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,1024,0.006301333506902059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,3072,0.07453155517578125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,2048,0.053716444306903414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,768,0.0052435555391841466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,1536,0.043634666336907275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,1024,0.03370489014519586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,512,0.0041893331540955436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,256,0.0030826667530669104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,128,0.002509333400262727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,64,0.002120888895458645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,512,0.02388444377316369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,768,0.028935111231274072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,256,0.018952000472280715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2048,32,32,0.0020906666500700847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,64,0.014868444866604276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,32,0.014337778091430664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,128,0.016241777274343703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,64,65536,5.908550262451172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,10240,2.810828526814779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,12288,3.2800089518229165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2048,32,65536,5.8942353990342875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,16384,4.3208872477213545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,10240,5.254583994547526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,8192,2.2400639851888022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,8192,4.149896833631727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,7168,2.157175064086914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,12288,6.150624169243707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,7168,3.617918226453993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,6144,1.7502835591634114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,6144,3.0936614142523875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,5120,1.4847342173258464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,4096,1.2528426912095811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,16384,8.138252258300781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,4096,2.0043040381537542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,5120,2.577686309814453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,3584,1.1179404788547092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,3584,1.76201597849528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,3072,0.9849155214097766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,3072,1.5034906599256728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,2048,0.7839742236667209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,2048,0.979066636827257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,2560,1.1626141866048176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,2560,0.9353164037068685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,1024,0.5001582039727105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,512,0.38089421060350204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,1536,0.7181102434794108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,768,0.4850444263882107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,512,0.46183109283447266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,1024,0.5877457724677192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,256,0.27599644660949707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,128,0.26499999894036186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,1536,0.6667528682284884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,768,0.5154479874504937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,64,0.21862488322787813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,256,0.4204097853766547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,64,0.3459635575612386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,128,0.3801040119594998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,65536,32,0.2638968891567654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,65536,32,0.3586551083458795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,10240,0.8422604666815864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,12288,1.0651404062906902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,10240,1.2890230814615886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,16384,1.431565284729004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,12288,1.5363040500217016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,16384,2.0573626624213324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,8192,0.6942488882276746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,7168,0.6317671140034994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,8192,1.0163404676649306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,7168,0.8911404079861112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,6144,0.7545955446031359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,6144,0.5761289066738552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,5120,0.6359004444546169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,4096,0.5243342187669542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,3584,0.45569509930080837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,5120,0.4726551373799642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,3584,0.35692887836032444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,4096,0.4127982192569309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,3072,0.40832000308566624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,2560,0.3282746738857693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,2560,0.27823999192979604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,2048,0.23945244153340658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,1536,0.2115537722905477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,3072,0.3314559989505344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,1024,0.12435111734602187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,1536,0.20176888836754692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,2048,0.24073955747816297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,512,0.07291555404663086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,768,0.09950577550464207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,256,0.040604445669386126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,768,0.1524346669514974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,128,0.026668444275856018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,512,0.1304711103439331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,1024,0.17085688643985322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,64,0.022312889496485393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,256,0.1121164427863227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,32,0.018218666315078735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,128,0.10681688785552979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,32,0.09998844729529487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,64,0.09647911124759251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,12288,1.1471200519137912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,16384,1.1647262573242188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,16384,1.4774808883666992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,12288,0.8579529126485189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,16384,65536,6.591820610894097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,10240,0.9396720462375217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,10240,0.7681200239393445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,8192,0.7230781979031033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,7168,0.5064489046732584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,6144,0.5256604618496364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,7168,0.6707244449191623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,8192,0.543774233924018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,6144,0.4512729114956326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,5120,0.4292728900909424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,4096,0.4543902079264323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,5120,0.4075306786431207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,4096,0.3200142118665907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,16384,65536,8.234905666775173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,3584,0.311302211549547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,2560,0.22037243843078613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,3072,0.25642845365736217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,3584,0.2932853433820936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,3072,0.30354399151272243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,2048,0.17760445011986625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,1536,0.14935110674964056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,2560,0.22671288914150664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,1024,0.107040888733334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,768,0.07444622119267781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,1536,0.15971733464135066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,512,0.062373333507113986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,2048,0.1877795590294732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,256,0.03253422180811564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,1024,0.1374959945678711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,128,0.02081155611409081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,512,0.10289688905080159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,64,0.018365333477656048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,32,0.014031999640994601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,128,0.08453689018885295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,64,0.07640088929070367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,256,0.09586755434672038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,768,0.11770221922132705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,32,0.07862044705284967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,12288,0.738791995578342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,12288,65536,5.6153369479709205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,12288,0.8630497720506457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,16384,1.2462737825181749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,16384,1.0833519829644096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,12288,65536,6.324230194091797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,8192,0.6011946466233995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,8192,0.5008542272779677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,10240,0.7282400131225586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,7168,0.6819911003112793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,10240,0.6223066647847494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,7168,0.4422889020707872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,6144,0.37185332510206437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,5120,0.34096267488267684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,5120,0.4887830946180556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,4096,0.37358845604790586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,4096,0.28034133381313747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,6144,0.5240435600280762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,3072,0.2879769007364909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,3584,0.2554808987511529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,3072,0.22897866037156847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,3584,0.30112266540527344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,2560,0.19410399595896402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,2048,0.19257956080966523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,1536,0.16641688346862793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,2048,0.16112355391184488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,2560,0.23314044210645887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,768,0.07875644498401217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,1536,0.1352248854107327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,1024,0.09987911250856187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,512,0.0553653339544932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,768,0.09974400202433269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,256,0.027293332748942908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,128,0.018208889497650992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,512,0.09080355697207981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,32,0.012671111358536614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,1024,0.11033244265450372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,256,0.07708089219199286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,128,0.07122044430838691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,64,0.015511999527613321
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,32,0.06685244374805026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,64,0.06727911366356744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,16384,0.9929013782077365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,12288,0.7072621981302897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,16384,0.9987582100762261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,10240,65536,4.910643683539496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,10240,65536,5.110041724310981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,10240,0.6086755328708225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,8192,0.5224400096469456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,12288,0.7301946745978461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,10240,0.5880915323893229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,7168,0.44935199949476456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,6144,0.33802400694953066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,8192,0.4703653123643663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,6144,0.3758560021718343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,7168,0.4429466724395752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,3584,0.20312266879611543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,4096,0.2347351180182563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,4096,0.28219734297858345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,5120,0.30325243208143443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,3072,0.1828977796766493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,5120,0.34195910559760195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,2048,0.13421600394778782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,3584,0.2887422243754069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,2560,0.14321866300370958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,3072,0.25811733139885795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,1536,0.10577955510881211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,2560,0.23148798942565918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,1024,0.06432977649900648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,768,0.05563022030724419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,2048,0.16854400104946562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,512,0.037216888533698186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,256,0.022657778528001573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,1024,0.12442933188544379
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,1536,0.1674586666954888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,768,0.11763821707831489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,64,0.013008000122176277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,512,0.10817333062489827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,32,0.009964444571071202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,256,0.10237510999043782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,64,0.09299200110965306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,128,0.09331644243664211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,32,0.08997066815694173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,128,0.015087111128701104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,8192,65536,4.33601337009006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,12288,0.6562808884514703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,16384,0.8254719840155708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,8192,65536,4.571414947509766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,12288,0.6877564324273003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,16384,0.9909689161512586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,10240,0.5487902429368761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,8192,0.5214942296346029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,10240,0.6001342137654623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,7168,0.38817334175109863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,8192,0.443195554945204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,6144,0.3703413274553087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,7168,0.3937413427564833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,5120,0.31285156144036186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,4096,0.25015200508965385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,3584,0.21365688906775582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,3072,0.1980231073167589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,5120,0.3357573350270589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,3584,0.2380524476369222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,6144,0.39077422353956437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,3072,0.235189331902398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,4096,0.2720115449693468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,2048,0.12607999642690024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,2560,0.1706897815068563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,1536,0.09973955816692776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,2560,0.20524799823760986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,2048,0.1552684439553155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,768,0.049907555182774864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,512,0.036320000886917114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,1024,0.11110222339630127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,256,0.02252977755334642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,1536,0.14927111731635198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,512,0.08855466710196601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,768,0.09883289204703437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,1024,0.07026577658123441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,128,0.015112888481881885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,64,0.011546666423479715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,256,0.0834159983528985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,32,0.009519110951158736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,64,0.07554044326146443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,32,0.07715821928448148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,128,0.08167911238140531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,12288,0.5173084470960829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,16384,0.8241590923733182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,7168,65536,3.8687731424967446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,12288,0.6797182295057508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,16384,0.9767173131306967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,10240,0.49272802140977645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,10240,0.5623795721266005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,7168,0.38827021916707355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,8192,0.4670426580641005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,8192,0.4551084306504991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,7168,0.4252311123741998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,7168,65536,4.53346210055881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,6144,0.36975733439127606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,5120,0.29230666160583496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,4096,0.22967555787828234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,6144,0.3663191000620524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,5120,0.32261066966586643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,3584,0.19963467121124268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,4096,0.2568151156107585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,3072,0.172071999973721
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,3072,0.19715732998318145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,2048,0.12332444720798069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,2560,0.15012266900804308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,2560,0.1945493353737725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,1024,0.05271555648909675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,2048,0.1432408889134725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,1536,0.08397777875264485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,3584,0.229423099093967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,768,0.040998223755094744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,1536,0.12161155541737874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,256,0.018207111292415194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,1024,0.0991706649462382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,512,0.03094844354523553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,768,0.08860622511969672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,128,0.012406222522258759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,64,0.009977777798970541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,512,0.08131200075149536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,32,0.00830666638082928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,256,0.0766542222764757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,128,0.0698195563422309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,64,0.06622133652369182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,32,0.06821599933836195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,6144,65536,3.5192142062717013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,12288,0.4429599973890517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,16384,0.6259733306037055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,10240,0.42664533191257054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,16384,0.9496942096286349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,12288,0.6632097562154134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,8192,0.3402053250206842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,7168,0.2952595551808675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,10240,0.548633787367079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,6144,65536,4.553793589274089
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,8192,0.45414490169949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,7168,0.4047751161787245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,5120,0.21748977237277559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,4096,0.20815111531151664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,6144,0.24658221668667266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,3584,0.16596978240542942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,5120,0.3113315635257297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,6144,0.37207910749647355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,4096,0.25211644172668457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,3072,0.15149688720703125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,3584,0.22303377257453072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,2048,0.09055378039677937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,3072,0.20658222834269205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,1536,0.09203288952509563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,2560,0.145416882303026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,1024,0.06885600090026855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,2048,0.14604266484578451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,768,0.04014755619896783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,1536,0.11664622359805638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,512,0.03045066528850132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,2560,0.16200978226131862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,1024,0.09407022264268662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,768,0.08709777726067437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,128,0.012093333734406365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,256,0.018479110466109384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,512,0.07501688930723402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,64,0.010023111270533668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,65536,2.758594724867079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,5120,32,0.008401778009202745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,128,0.06488444407780965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,256,0.07324355840682983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,64,0.060777776771121554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,32,0.06135288874308268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,12288,0.33526134490966797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,10240,0.32565511597527397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,16384,0.4970195558336046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,12288,0.46368710199991864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,16384,0.6794257693820529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,8192,0.2851662105984158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,8192,0.32929245630900067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,10240,0.38736266560024685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,7168,0.1962613397174411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,6144,0.2081600030263265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,7168,0.26322756873236763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,4096,0.14089600245157877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,5120,0.18674755096435547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,6144,0.23325067096286348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,3584,0.11993600262535943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,4096,0.12835466861724854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,3072,0.09171111053890652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,5120,0.1504488918516371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,2560,0.07784177859624226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,3072,0.1131324503156874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,2560,0.0889786680539449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,2048,0.0632257792684767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,3584,0.11525689231024848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,1536,0.053519109884897866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,1024,0.03278755479388767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,2048,0.07473511166042753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,768,0.03223911258909438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,512,0.020957332518365648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,1536,0.06574133369657728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,1024,0.05410933494567871
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,256,0.012974222501118978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,5120,65536,4.523359086778429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,768,0.04911466770701938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,512,0.04391733474201626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,128,0.008406221866607666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,64,0.006826666494210561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,65536,2.2160587310791016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,4096,32,0.00598933340774642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,256,0.03811466693878174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,64,0.03198666705025567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,32,0.031883554326163396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,128,0.034288889831966825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,12288,0.338626676135593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,16384,0.44148000081380206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,16384,0.6745475663079156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,10240,0.29300888379414874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,12288,0.47019020716349286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,10240,0.3784968852996826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,8192,0.23851911226908365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,7168,0.1925333340962728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,8192,0.2990800009833442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,6144,0.16370133558909097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,5120,0.14079911179012722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,6144,0.2175288862652249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,7168,0.25915733973185223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,5120,0.1798106696870592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,4096,65536,3.7027672661675344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,3584,0.10078310966491699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,4096,0.11352711253696018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,65536,1.9387982686360676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,3072,0.08804444471995036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,2560,0.07491644223531087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,4096,0.12574933634863958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,3072,0.09716000159581502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,3584,0.11063822110493977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,2560,0.10280088583628337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,1536,0.04493511054250929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,1024,0.0338986681567298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,768,0.028031110763549805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,2048,0.07278311252593994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,512,0.020080889264742535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,2048,0.060065779421064586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,768,0.04725155565473768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,256,0.01220355596807268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,1536,0.06297600269317627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,512,0.04198933309978909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,64,0.00647555540005366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,1024,0.055382221937179565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,32,0.005727111051479976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3584,128,0.008198222352398766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,128,0.032136887311935425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,64,0.03021866745418972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,256,0.03666222095489502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,32,0.02979733215437995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,10240,0.22135733233557808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,12288,0.2628142303890652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,16384,0.3438177903493245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,8192,0.18642044067382812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,16384,0.673125320010715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,12288,0.48383198844061953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,10240,0.36698044670952695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,8192,0.2929520077175564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,7168,0.1860648923450046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,6144,0.16414577431148952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,5120,0.13944178157382542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,4096,0.12481689453125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,7168,0.25073422325981987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,6144,0.2131484482023451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,5120,0.17373243967692056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3584,65536,3.6862229241265188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,3584,0.10976088709301418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,4096,0.1226773394478692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,65536,1.652032004462348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,3584,0.10967555310991074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,3072,0.10326933198504978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,2048,0.05769955449634128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,1536,0.04446755515204536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,3072,0.09391378031836616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,2048,0.08280266655815972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,2560,0.09296266900168525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,2560,0.08200266626146105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,1536,0.060769776503245033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,768,0.027266666293144226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,1024,0.03348177671432495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,256,0.011857777833938599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,1024,0.05005066593488058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,512,0.019492444064882066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,128,0.008044444852405125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,64,0.006182222316662471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,512,0.03986755675739712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,3072,32,0.005264889034960005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,256,0.03461066550678677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,768,0.04908533228768242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,64,0.029232889413833618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,128,0.030709332889980737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,32,0.028618666860792372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,10240,0.2176133394241333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,16384,0.3020382192399767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,12288,0.2322924402025011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,8192,0.1682142284181383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,12288,0.458205329047309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,16384,0.6845964325798882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,10240,0.3681653340657552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,7168,0.15890844662984213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,8192,0.2867208851708306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,6144,0.1435813373989529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,7168,0.2495680120256212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,5120,0.14153244760301378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,6144,0.20627200603485107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,4096,0.1022168861495124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,65536,1.3738329145643446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,5120,0.1694115532769097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,4096,0.12526755862765843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,3584,0.08984800179799397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,3072,0.09127110905117458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,3584,0.10977333121829563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,2560,0.07147822115156385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,1536,0.04914044340451559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,2560,0.07980622185601129
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,2048,0.08039999670452543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,3072,65536,3.6727892557779946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,2048,0.0708471139272054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,1024,0.03397066725624932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,3072,0.09205955266952515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,1536,0.06012977494133843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,768,0.026739555928442214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,256,0.011732444167137146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,768,0.04409511221779717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,1024,0.051382223765055336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,512,0.019667555888493855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,512,0.03849688834614224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,128,0.008075555165608725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,32,0.004933333231343163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2560,64,0.0064151109092765385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,128,0.030080888006422255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,64,0.027581334114074707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,32,0.027230223019917805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,256,0.03350044290224711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,12288,0.1883280012342665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,16384,0.258499542872111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,10240,0.16405599647098118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,8192,0.13527645005120173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,10240,0.2920960055457221
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,12288,0.37454666031731504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,16384,0.5914959907531738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,7168,0.1076657772064209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,8192,0.22622667418585884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,6144,0.09381777710384792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,5120,0.08119466569688585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,65536,0.9579590691460503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,7168,0.19317955440945098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,4096,0.06153421931796604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,3584,0.06093155675464206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,5120,0.12479466862148708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,6144,0.1552106671863132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,3072,0.04535733328925239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,4096,0.0864844454659356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,2560,0.04019999835226271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,2048,0.03154311246342129
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,3584,0.0747528870900472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,1536,0.027971555789311726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,3072,0.07201155689027575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,2560,0.054970665110482105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,1024,0.019911999503771465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,2048,0.04940977692604065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,512,0.011737777955002256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,1024,0.03489066825972663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,1536,0.0417119993103875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,256,0.00756533361143536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,128,0.005537777725193236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,768,0.03150666753451029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,512,0.02864711152182685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,768,0.018058665924602084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,64,0.004578666554556953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,2048,32,0.003942222230964237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,128,0.02147644427087572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,256,0.024102222588327196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,64,0.020118221640586853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2560,65536,3.668762630886502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,32,0.019905777441130746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,12288,0.14801422754923502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,16384,0.21463110711839464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,10240,0.12427822748819987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,8192,0.11713955137464736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,7168,0.10526044501198663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,12288,0.35547465748257107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,8192,0.19711377885606554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,16384,0.5816702312893337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,10240,0.2681564490000407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,6144,0.0851644476254781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,5120,0.07215378019544813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,65536,0.8158364295959473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,7168,0.16934488879309761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,4096,0.058000889089372426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,5120,0.10347911384370591
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,3584,0.06120977799097697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,6144,0.13262755341000027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,4096,0.0745333300696479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,3072,0.04440711273087395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,3584,0.06688621971342298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,2560,0.04527911212709215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,2048,0.0309804446167416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,3072,0.058746669027540416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,2560,0.052410665485594005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,1024,0.018059555027219985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,2048,0.04503733250829908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,1536,0.02875111169285244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,768,0.014863111906581454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,1536,0.038607110579808555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,512,0.01145511120557785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,256,0.007117333511511485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,128,0.005251555393139522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,768,0.028788444068696763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,512,0.02555022140343984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,64,0.004439110971159405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,1024,0.03249866763750712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1536,32,0.0037288889288902283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,256,0.02295733326011234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,128,0.019194665882322524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,64,0.01844800015290578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,2048,65536,3.2095262739393444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,32,0.017991110682487488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,16384,0.1251386668947008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,10240,0.08258399698469374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,12288,0.09550133016374375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,8192,0.07306489017274645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,12288,0.32429422272576225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,65536,0.5393582450018989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,10240,0.2475057707892524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,7168,0.07193599806891547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,16384,0.574144893222385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,6144,0.051199999120500356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,7168,0.14021066824595133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,8192,0.1743911107381185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,5120,0.044122666120529175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,6144,0.11044444640477498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,4096,0.03667022122277154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,3072,0.03650577863057455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,3584,0.032976888948016696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,5120,0.08692977825800578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,2560,0.027735999888843958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,4096,0.0702400008837382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,3584,0.06321599748399523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,2048,0.02232088810867733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,3072,0.06383199824227227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,1536,0.017613333132531907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,2560,0.04990577697753906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,1024,0.01348444405529234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,2048,0.04301955633693271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,768,0.010238221950001186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,1536,0.036441776487562395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,1024,0.029764443635940552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,256,0.005715555614895291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,128,0.00453333349691497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,768,0.02854666776127285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,512,0.023353776997990076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,64,0.0038177776667806837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,512,0.007822222179836696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,1024,32,0.003731555408901639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,256,0.019462222854296368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,128,0.017071111334694754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,64,0.016167999969588388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1536,65536,3.171951082017687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,32,0.016136000553766888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,16384,0.09635644488864475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,10240,0.0697279969851176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,12288,0.0804719991154141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,8192,0.05482666691144308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,65536,0.44412533442179364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,7168,0.04769866665204366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,10240,0.22203644116719565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,12288,0.31101422839694554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,6144,0.04279822111129761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,8192,0.1616106695599026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,5120,0.03661511010593838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,4096,0.03064799971050686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,16384,0.5517439842224121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,7168,0.13648088773091635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,3584,0.031288888719346784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,5120,0.08276000287797716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,3072,0.02936089038848877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,6144,0.10279555453194512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,2560,0.022831110490692988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,4096,0.06865333186255561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,3584,0.0618399977684021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,2048,0.017833777599864535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,3072,0.05503999855783251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,1536,0.014783999986118741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,2560,0.050776888926823936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,1024,0.010483555495738983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,768,0.008857777549160851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,2048,0.04166488846143087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,512,0.006864000111818314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,1536,0.03532888823085361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,256,0.005144000053405762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,1024,0.028563555743959215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,128,0.003912888881233004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,768,0.025861332813898723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,64,0.0032844444115956626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,512,0.022247110803922016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,256,0.018301332990328472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,768,32,0.00309244439833694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,128,0.016255110502243042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,64,0.015620445211728414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,32,0.015180443723996481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,16384,0.07118222448560926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,12288,0.05038311084111532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,10240,0.04143288731575012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,65536,0.35232088300916886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,1024,65536,3.170867496066623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,8192,0.034508443540996976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,12288,0.30461422602335614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,7168,0.03131466772821214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,10240,0.21940088272094727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,16384,0.5535715421040853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,6144,0.027867555618286133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,5120,0.024659555819299486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,7168,0.11892799536387126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,8192,0.15184444851345485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,4096,0.020600888464185927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,3584,0.01924000018172794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,5120,0.08159999714957343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,6144,0.09709866841634114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,3072,0.01706044375896454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,4096,0.0677200025982327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,2560,0.015663110547595553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,3584,0.061412447028689914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,2048,0.01572444372706943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,3072,0.05444000164667765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,1536,0.012390221986505719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,2560,0.04788088798522949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,2048,0.041103111373053655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,768,0.0074942219588491656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,1024,0.008954666554927826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,1536,0.034508443540996976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,512,0.005984888722499211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,1024,0.02786933382352193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,256,0.004420444369316101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,128,0.0036959999965296853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,768,0.024711999628278945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,64,0.0029360000044107437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,512,0.021128889587190416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,256,0.017595556047227647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,512,32,0.002760888801680671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,128,0.015449777245521545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,64,0.014532445205582513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,16384,0.034493333763546415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,32,0.014150222142537435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,12288,0.027394667267799377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,10240,0.023764444722069636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,65536,0.27435376909044057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,8192,0.02017333275742001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,768,65536,3.1676027509901257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,12288,0.2905297809176975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,10240,0.21008977625105116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,7168,0.019070222973823547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,6144,0.017338666650984023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,5120,0.016922665966881644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,16384,0.5446097585890028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,8192,0.13689066304100886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,7168,0.11034666829639012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,4096,0.014202665951516895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,6144,0.09505066606733535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,3584,0.013630222115251752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,3072,0.012286221815480126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,2560,0.011196444431940714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,5120,0.08097777764002483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,2048,0.009556444154845344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,4096,0.06769955820507474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,3584,0.06104533539877998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,3072,0.05421955717934502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,1536,0.008405333591832055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,1024,0.006362666686375936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,768,0.005590222362014983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,2048,0.040778666734695435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,2560,0.04780711068047417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,1536,0.034536888202031456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,512,0.004591111093759537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,256,0.003585777762863371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,1024,0.02754755483733283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,128,0.003069333318206999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,768,0.024460444847742718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,64,0.002516444358560774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,512,0.020960888928837244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,256,0.01737955543729994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,256,32,0.0022622222701708474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,128,0.01555022266176012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,64,0.014521777629852295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,16384,0.023614222804705303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,32,0.014184888866212634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,12288,0.019310222731696237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,10240,0.016732444365819294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,8192,0.016521778371598985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,65536,0.24563821156819662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,512,65536,3.1751929389105906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,7168,0.01532799998919169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,6144,0.013912889692518445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,12288,0.283315552605523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,10240,0.20005333423614502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,8192,0.13219911522335476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,5120,0.012168889244397482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,7168,0.10963733328713311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,6144,0.09564444753858779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,16384,0.5510986646016439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,5120,0.08086578051249187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,4096,0.01014311114947001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,4096,0.06767555740144518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,3584,0.06128888660007053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,3584,0.009656888743241629
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,3072,0.00868888861603207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,3072,0.054437332683139376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,2048,0.011530666715568967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,1536,0.00944088896115621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,2560,0.007915555602974361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,1024,0.006816000160243776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,2048,0.040765331851111516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,2560,0.04792444573508369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,768,0.005690666536490123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,1536,0.03448977735307481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,1024,0.027683556079864502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,256,0.003228444399105178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,512,0.004384888956944148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,128,0.0025724443710512584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,768,0.024392889605628118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,64,0.0021866667601797315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,512,0.02108977735042572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,128,32,0.0020346666375796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,128,0.01571644511487749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,256,0.017384888397322763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,64,0.014499555031458536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,32,0.014295111099878946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,16384,0.031097776359981958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,12288,0.025777777036031086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,10240,0.022022222479184467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,65536,0.20785155561235216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,8192,0.01740977830357022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,12288,0.27561155954996747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,256,65536,3.188987520005968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,10240,0.19752711719936797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,8192,0.12861955165863037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,7168,0.01665422154797448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,5120,0.013099555340078143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,6144,0.014843554960356818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,16384,0.5638631184895834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,4096,0.01092888911565145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,3584,0.010410666465759277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,7168,0.1102328896522522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,6144,0.09566755427254571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,5120,0.08082044124603271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,3072,0.00941244430012173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,4096,0.06727288828955756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,2560,0.00849599970711602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,2048,0.0074986666440963745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,3072,0.05423288875155979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,1536,0.008866666919655269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,3584,0.06090311209360758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,2560,0.04746133420202467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,2048,0.040764444404178195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,1024,0.0064995553758409284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,768,0.005320000151793162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,512,0.0041475556790828705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,256,0.003007111036115222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,1536,0.034338666333092585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,1024,0.027552000350422327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,128,0.0024195555597543716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,768,0.02437244521247016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,512,0.020752888586786058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,64,0.0020826665891541377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,256,0.017126222451527912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,64,32,0.0018240000224775737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,128,0.015519110692871941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,64,0.014475555883513557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,16384,0.01739288866519928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,32,0.014124444789356656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,12288,0.014653333359294467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,10240,0.012866666747464074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,65536,0.20256443818410239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,8192,0.010753778119881948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,7168,0.011863110793961419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,10240,0.1976675589879354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,12288,0.2743271191914876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,8192,0.1274364391962687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,6144,0.01053955571519004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,128,65536,3.177228503757053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,7168,0.10998133156034683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,5120,0.009398221969604492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,3584,0.007613333563009898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,4096,0.00810933361450831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,16384,0.5525884628295898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,6144,0.09525244765811497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,5120,0.0806293355094062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,3072,0.007012444237867991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,4096,0.06723555591371325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,2560,0.006362666686375936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,3584,0.06087911128997803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,2048,0.005736888696750005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,1536,0.007796444826655918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,1024,0.005793777604897817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,2560,0.047372443808449634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,3072,0.05423022309939066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,2048,0.04055555661519369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,768,0.004832889056868023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,1536,0.03435466686884562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,512,0.0038168889780839286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,256,0.002986666642957263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,1024,0.02757688860098521
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,128,0.0023191111783186593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,768,0.02432800001568264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,512,0.020899555749363367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,64,0.0019946667469210094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,256,0.017123555143674214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1024,32,32,0.0022088889446523455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,128,0.015583111180199517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,64,0.014570666684044732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,32,0.014306666122542487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,64,65536,3.186751047770182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1024,32,65536,3.1858435736762156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,12288,2.455307642618815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,10240,2.045079125298394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,16384,3.2920203738742404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,10240,3.9266632927788625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,8192,1.7072381973266602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,7168,1.484214252895779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,12288,4.657542334662543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,8192,3.1411590576171875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,7168,2.670889748467339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,6144,1.316215091281467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,6144,2.345277362399631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,5120,1.9500577714708116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,5120,1.1033030615912545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,16384,6.184790293375651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,4096,0.9631449381510416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,4096,1.5095236036512587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,3584,0.8371084531148275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,3584,1.309292475382487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,3072,0.7936124271816678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,3072,1.1297849019368489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,2560,0.687128914727105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,2560,0.8908231523301867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,1536,0.5748017628987631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,2048,0.745231999291314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,2048,0.5956711239284939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,1024,0.4542871051364475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,768,0.3710906770494249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,512,0.35730844073825413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,256,0.2692062324947781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,1024,0.42054398854573566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,768,0.4050559997558594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,1536,0.5070728725857204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,512,0.3753608862559001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,128,0.20327377319335938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,64,0.1741795539855957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,256,0.3174497816297743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,128,0.28788354661729604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,64,0.28397154808044434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,65536,32,0.16651644971635607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,65536,32,0.2657990985446506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,10240,0.6302222145928277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,12288,0.7570444213019477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,10240,0.8873946401807996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,12288,1.1161218219333224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,16384,1.0540071063571506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,16384,1.4985004001193578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,8192,0.50564087761773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,7168,0.6722008917066785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,6144,0.5623155699835883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,7168,0.4871066411336263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,8192,0.7154080073038737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,6144,0.46117687225341797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,5120,0.45192888047960067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,4096,0.3814817799462213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,5120,0.37768088446723086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,3584,0.3354088995191786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,4096,0.2937866581810845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,3072,0.30834934446546763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,3072,0.2583422131008572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,3584,0.27075910568237305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,2560,0.2171013355255127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,1536,0.148035552766588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,2048,0.17692622873518202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,2560,0.239391114976671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,1536,0.1532720062467787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,768,0.073725336127811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,2048,0.2031386693318685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,1024,0.1330640051099989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,256,0.033181332879596286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,1024,0.09601333406236436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,768,0.11747111214531793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,512,0.10142666763729519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,128,0.020947555700937908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,512,0.056391113334231906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,64,0.01721955504682329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,32,0.014319111903508505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,256,0.09234578079647487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,64,0.07595111264122857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,128,0.08179199695587158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,32,0.07773066891564263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,16384,0.8667244381374783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,12288,1.1056257883707683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,16384,1.4769084718492296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,16384,65536,4.950643751356337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,12288,0.6329769028557671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,10240,0.8184293111165365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,10240,0.5656951268513998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,8192,0.7016755739847819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,8192,0.43138400713602704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,7168,0.40178489685058594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,7168,0.6188737551371256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,6144,0.5781084696451823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,6144,0.32954666349622935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,5120,0.47173065609402126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,5120,0.3053199979994032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,4096,0.3655431005689833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,16384,65536,6.9998567369249125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,4096,0.24663199318779838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,3584,0.3082257906595866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,3584,0.22964623239305285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,3072,0.2759022182888455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,2560,0.2621271080440945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,2560,0.16885156101650664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,2048,0.18835822741190592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,2048,0.1487306621339586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,1024,0.10109333197275798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,3072,0.1900026665793525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,768,0.07759910821914673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,512,0.04649510979652405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,1536,0.14533510473039415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,768,0.09147733449935913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,512,0.08193688922458224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,256,0.026826666461096868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,128,0.016833777228991192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,1536,0.1274284389283922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,1024,0.10134666495853001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,64,0.01511111193233066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,256,0.06760799884796143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,128,0.06307644314236112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,32,0.012091555529170565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,64,0.0588302214940389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,32,0.05828711059358385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,12288,65536,4.430462307400174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,12288,0.5421733326382107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,12288,0.7050871319240994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,10240,0.5904577573140463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,16384,0.7833369043138293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,16384,1.0488462448120117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,12288,65536,5.297768910725911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,10240,0.46058400472005206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,8192,0.39655645688374835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,6144,0.34845156139797634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,7168,0.4057182206047906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,7168,0.3557573424445258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,8192,0.4617413414849176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,6144,0.29933688375684947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,5120,0.24306133058336046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,4096,0.21692000495062935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,4096,0.2413946787516276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,5120,0.3012097676595052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,3584,0.21100800567203096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,3072,0.1711119943194919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,2560,0.14749599827660456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,2048,0.12033422787984212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,2560,0.1393315527174208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,3072,0.1678560045030382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,3584,0.18456621964772543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,1024,0.06676799721188016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,768,0.04955288767814636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,1024,0.08853422270880805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,512,0.03600000010596381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,2048,0.12277955479092068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,256,0.021917333205540974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,768,0.07945955461925931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,1536,0.08959555625915527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,512,0.07124444511201647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,1536,0.10694044166141087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,256,0.061760001712375216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,64,0.012657778130637275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,128,0.014863111906581454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,32,0.010048888623714447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,128,0.056249777475992836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,64,0.05423555440372891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,32,0.05310933457480537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,12288,0.597000863817003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,12288,0.7408400111728244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,16384,0.7675093544854058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,16384,0.830477343665229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,10240,65536,3.9617237514919705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,8192,0.43440622753567165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,10240,0.5491315523783366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,10240,0.5210204654269749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,8192,0.4082382255130344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,10240,65536,4.441046396891276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,6144,0.34350400500827366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,7168,0.3904799885219998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,5120,0.2958151234520806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,6144,0.3353297710418701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,5120,0.2779902352227105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,7168,0.380037334230211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,3584,0.21240799956851533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,3072,0.18909688790639242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,4096,0.23317599296569824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,3584,0.2017191118664212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,4096,0.22605511877271864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,2048,0.13038578298356798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,2560,0.157641781700982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,2048,0.13282844755384657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,3072,0.20392089419894746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,2560,0.16131556034088135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,1536,0.08455733458201091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,1024,0.053118222289615206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,768,0.04146044452985128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,1536,0.12874221801757812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,1024,0.0988888872994317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,768,0.08748444583680894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,256,0.017878222796652053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,128,0.012480888929631976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,512,0.08214844597710504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,64,0.00998933364947637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,256,0.0737066666285197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,512,0.02920088834232754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,32,0.008362666600280339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,128,0.06963022549947102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,64,0.06884177525838216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,32,0.06685155630111694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,8192,65536,3.5347947014702688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,12288,0.445200867123074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,16384,0.5930879910786947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,8192,65536,3.851239946153429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,10240,0.3697306580013699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,12288,0.602459536658393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,16384,0.8310417599148221
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,10240,0.5081466568840874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,8192,0.3281502193874783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,6144,0.2692106564839681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,7168,0.3706195619371202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,5120,0.2331999937693278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,7168,0.32380443149142796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,8192,0.437084436416626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,3584,0.1562088860405816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,4096,0.2066871060265435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,6144,0.33853332201639813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,3584,0.20010044839647081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,4096,0.22826843791537813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,3072,0.13616889052920872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,5120,0.27492088741726345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,3072,0.2089031140009562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,1536,0.08888533380296494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,2048,0.1225093338224623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,2560,0.1227288908428616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,2560,0.15522044234805638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,1024,0.05055377880732218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,768,0.039255112409591675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,512,0.02940888868437873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,256,0.01829955478509267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,768,0.08578399817148845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,2048,0.13641422324710423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,1536,0.12603644529978433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,1024,0.09607111083136664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,128,0.01218311074707243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,512,0.07470844189325969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,64,0.01001422190003925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,32,0.008240000241332585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,256,0.0707919994990031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,128,0.06630755795372857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,64,0.06342755423651801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,32,0.06173066960440742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,12288,0.4647199842664931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,7168,65536,3.345969729953342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,16384,0.6268826590643989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,12288,0.5934951040479872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,16384,0.8045351240370008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,8192,0.34263910187615293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,7168,65536,3.8573892381456165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,10240,0.3974622090657552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,10240,0.4992364247639974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,7168,0.3210746712154812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,6144,0.3145964410569933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,8192,0.4081439971923828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,5120,0.24271999465094674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,6144,0.3352168930901422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,7168,0.37801067034403485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,5120,0.27276266945732963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,4096,0.22559287812974718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,4096,0.22269778781467012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,3584,0.1779315604103936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,3072,0.17106578085157606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,3584,0.19743111398484972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,2560,0.12465422683291966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,2048,0.1096328894297282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,1536,0.08225422435336642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,2048,0.13072978125678167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,2560,0.15568355719248453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,1024,0.050944000482559204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,3072,0.1994702286190457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,768,0.03849422269397312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,512,0.027722666660944622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,256,0.01683022247420417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,768,0.08035644557740954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,128,0.011696889168686338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,1024,0.08993599812189738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,1536,0.10746933354271783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,512,0.07061066892411974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,64,0.009575111170609793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,32,0.00775911079512702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,256,0.06580622328652276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,64,0.054754667811923556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,128,0.05929955508973864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,32,0.05425511134995354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,6144,65536,2.8594915601942272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,12288,0.3384568956163194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,16384,0.44965330759684247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,16384,0.562155564626058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,12288,0.41260356373257107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,10240,0.2990142239464654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,8192,0.22568978203667533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,7168,0.21284266312917074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,10240,0.3478079901801215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,6144,65536,3.8599154154459634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,8192,0.30303555064731175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,6144,0.23733332422044542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,5120,0.15152266290452746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,6144,0.17540444268120658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,4096,0.1132391161388821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,7168,0.26551734076605904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,3584,0.10046666860580444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,4096,0.14734666877322727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,5120,0.190247999297248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,3072,0.0879439976480272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,2048,0.05946133534113566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,3072,0.1301742262310452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,3584,0.12688355975680882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,1536,0.04572355416085985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,65536,2.210893419053819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,2560,0.10106933116912842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,2048,0.08199555344051786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,1536,0.06453244553671943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,2560,0.07478400071461995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,768,0.028348445892333984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,512,0.020065777831607394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,1024,0.03198400139808655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,1024,0.05138311121198866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,128,0.008352888955010308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,256,0.012704888979593912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,768,0.04783022072580126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,64,0.00664444433318244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,5120,32,0.0058204444746176405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,512,0.042325334416495435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,256,0.03771911064783732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,128,0.03329244587156508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,64,0.03436355458365546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,16384,0.35184444321526426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,12288,0.3018782138824463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,32,0.03227555420663621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,5120,65536,2.96742672390408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,16384,0.49086311128404403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,10240,0.2360515594482422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,8192,0.20614222685496011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,10240,0.32795998785230845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,12288,0.3835448953840468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,7168,0.19370667139689127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,6144,0.16946844259897867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,4096,0.11713333924611409
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,5120,0.14042488733927408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,8192,0.25809423128763836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,5120,0.1361573272281223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,3584,0.09922489192750718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,3072,0.08639822403589885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,4096,0.11513333850436741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,3584,0.09765599833594428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,7168,0.21072799629635283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,65536,1.5393786960177953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,6144,0.17709955904218885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,2560,0.07290844122568767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,2048,0.05788889196183947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,1024,0.03431199987729391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,2560,0.0788373351097107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,768,0.027657777070999146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,2048,0.06750577688217163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,1536,0.05815911293029785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,1024,0.04859111044141981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,512,0.019886222150590684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,1536,0.05113510953055488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,256,0.011980444192886353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,768,0.04457333352830675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,128,0.008097777764002482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,3072,0.08902933200200398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,64,0.006196444647179709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,4096,32,0.005271111097600725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,512,0.039140443007151283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,256,0.03382666574584113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,64,0.02901244494650099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,128,0.032799111472235784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,32,0.028526223368114893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,12288,0.25270220968458385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,16384,0.3058568901485867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,10240,0.20092177391052246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,8192,0.16842133469051787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,16384,0.5087466769748265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,12288,0.3771546681722005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,10240,0.3128746615515815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,8192,0.23881867196824816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,7168,0.16645867294735378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,7168,0.21098221672905815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,6144,0.1632959975136651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,5120,0.1367457840177748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,6144,0.17066400580936006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,4096,0.10477066702312893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,5120,0.13464621702829996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,4096,65536,2.9865430196126304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,3584,0.08568266365263198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,3072,0.08614310953352187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,3584,0.10332177744971381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,2560,0.06769422027799818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,2048,0.07912711302439372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,2560,0.07648799816767375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,4096,0.10853422350353664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,1536,0.04933599962128533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,65536,1.634480900234646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,1024,0.03416622347301907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,3072,0.08935644229253133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,2048,0.06658755408393012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,768,0.031164444155163232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,1536,0.05716355641682943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,256,0.013738666971524557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,1024,0.04889066682921516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,512,0.019193778435389202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,768,0.04336355461014641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,128,0.008738666772842407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,64,0.006128888991143968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,256,0.032983111010657415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,128,0.029902223083708022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,512,0.03804711169666714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,64,0.028417779339684382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3584,32,0.005389333185222413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,32,0.02758666707409753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,16384,0.3044817712571886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,12288,0.23495290014478895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,12288,0.37278132968478733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,10240,0.20105600357055664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,16384,0.5180880228678385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,8192,0.17315377129448783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,10240,0.3100364473130968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,7168,0.16519199477301702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,8192,0.24618132909138998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3584,65536,2.9790738423665366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,7168,0.20700177881452772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,6144,0.14402933915456137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,5120,0.1404177745183309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,65536,1.3621528413560655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,4096,0.10847644673453437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,3072,0.10706310802035862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,4096,0.0979928904109531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,6144,0.17685511377122667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,3584,0.09394755628373887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,2048,0.05029955506324768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,3584,0.10138577885097927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,2560,0.08281866709391277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,5120,0.1319955587387085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,3072,0.08591110838784112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,2560,0.07317866881688435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,1536,0.047650665044784546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,2048,0.06395466460122003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,768,0.026537777649031744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,512,0.019512888458040025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,1536,0.054099556472566396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,1024,0.0442808899614546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,256,0.011493333511882357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,1024,0.03736889031198289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,128,0.008002666963471307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,768,0.03977866636382209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,64,0.006044444524579578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,512,0.03516711129082574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,3072,32,0.00462666650613149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,256,0.029393778906928167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,128,0.026547556122144062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,64,0.026016889346970454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,32,0.025006221400366888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,12288,0.19723910755581328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,16384,0.23864621586269805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,10240,0.16349689165751138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,12288,0.2802995575798882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,8192,0.13181600305769178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,16384,0.3975848886701796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,8192,0.16901422871483696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,10240,0.22444444232516816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,7168,0.10653066635131836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,6144,0.08589510785208808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,5120,0.07932266924116346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,7168,0.14386755890316433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,6144,0.11597155200110541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,5120,0.08685333198971218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,3584,0.057922667927212186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,4096,0.058984888924492725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,4096,0.06934577888912626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,3072,0.047019554509056934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,2560,0.03921511107020908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,3584,0.06208178069856432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,2048,0.032368001010682844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,65536,1.0581546359592013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,1536,0.027264000640975103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,3072,0.05684533384111193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,2560,0.05079466766781277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,2048,0.04522844486766391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,3072,65536,2.992848926120334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,1024,0.0188080006175571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,768,0.015048000547620984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,256,0.007425777614116669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,1536,0.04342488778962029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,512,0.0110871113008923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,1024,0.03243466549449497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,512,0.02631644407908122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,128,0.006213333457708359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,64,0.004573333180612988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,768,0.030474666092130873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,256,0.0229093333085378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,64,0.019519999623298645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,32,0.0192586663696501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,128,0.020810667011472914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,16384,0.22043555312686494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,12288,0.16189155313703749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2560,32,0.003962666624122196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,10240,0.1371084451675415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,16384,0.3839013311598036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,8192,0.11710488796234131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,12288,0.25633955001831055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,65536,0.8233555687798394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,7168,0.1033324466811286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,8192,0.1473671065436469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,10240,0.21377600563897026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,6144,0.09784444173177083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,7168,0.12418933709462483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,5120,0.07263466384675768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,4096,0.06709422005547418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,6144,0.10398399829864502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,3584,0.05142133434613546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,3072,0.04588977826966179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,4096,0.06568355692757501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2560,65536,2.5043680402967667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,2560,0.03905155592494541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,5120,0.0809697773721483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,3072,0.05422044462627835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,2048,0.03100444542037116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,2560,0.04764266808827718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,1536,0.026325333449575637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,2048,0.04250311189227634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,3584,0.06371377574072944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,1024,0.018107554978794523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,1536,0.036383999718560114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,768,0.015096000499195524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,1024,0.030510223574108545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,512,0.011514667007658215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,256,0.007176888485749562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,768,0.027759111589855615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,128,0.0052844443255000645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,64,0.004411555412742826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,2048,32,0.003689777933888965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,256,0.021514667405022517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,128,0.020032000210550096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,512,0.02456444501876831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,64,0.01846577723821004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,32,0.018075555562973022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,12288,0.11725244257185195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,16384,0.1485511064529419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,10240,0.09982844193776448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,12288,0.23795821931627062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,8192,0.08305333058039348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,10240,0.18655110730065239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,16384,0.36690666940477157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,8192,0.13243733512030706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,7168,0.09680977794859146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,65536,0.592001756032308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,6144,0.0826408863067627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,4096,0.056129780080583364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,5120,0.07825155390633477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,7168,0.10971555444929336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,5120,0.07657866345511542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,3584,0.05059377683533562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,6144,0.09213866790135701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,2048,65536,2.475109312269423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,2560,0.041952000723944775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,3072,0.0398613346947564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,3584,0.058674666616651744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,1536,0.025900445050663416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,3072,0.052967998716566295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,2560,0.04689866635534498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,4096,0.06535555256737603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,1024,0.018062222335073683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,768,0.014675555957688225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,512,0.01070844464831882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,1536,0.03534577621353997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,2048,0.030086222622129653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,2048,0.04453600115246243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,1024,0.029581334855821397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,256,0.00703022215101454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,768,0.02720444401105245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,64,0.004303110970391167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,128,0.00517155561182234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,512,0.02351288828584883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1536,32,0.0031439999325407874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,256,0.020202666521072388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,64,0.01752622259987725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,128,0.019175999694400363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,32,0.01721155477894677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,16384,0.099199997054206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,12288,0.08138933446672228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,10240,0.06997155480914645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,8192,0.05372533202171326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,65536,0.4337795575459798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,12288,0.21736266877916124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,10240,0.17321511109670004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,7168,0.05372711022694906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,16384,0.3427920076582167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,6144,0.043007999658584595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,8192,0.11794488959842259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,5120,0.036171555519104004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,7168,0.10097333457734849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,4096,0.030693332354227703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,3584,0.030448890394634668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,5120,0.07382488913006253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,4096,0.061922669410705566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,3072,0.02566755480236477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,6144,0.09140444464153713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,3584,0.056029333008660205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,2560,0.0229111115137736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,2048,0.018408889571825664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,1536,0.014535110857751636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,3072,0.050208889775806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,2560,0.044287112024095326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,1024,0.010560000108347999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,2048,0.038711110750834145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,768,0.008671110702885522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,1536,0.03360088997417026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,512,0.006743999818960826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,1024,0.027049778236283198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,256,0.0052151112920708126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,768,0.024178667200936213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,128,0.00397155558069547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,512,0.021115556359291077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,64,0.003272888975010978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,1024,32,0.003080000066094928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1536,65536,2.464240815904405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,128,0.016161777906947665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,256,0.017785777648289997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,64,0.015184000134468079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,32,0.014841778410805596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,16384,0.07961422204971313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,12288,0.07064177592595418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,10240,0.05084088775846693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,8192,0.04173955652448866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,65536,0.3537448777092828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,12288,0.2193653318617079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,7168,0.03663555449909634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,10240,0.16242844528622097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,16384,0.3379973305596246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,6144,0.03752266698413425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,8192,0.11356177594926621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,5120,0.029859556092156306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,4096,0.025349333882331848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,7168,0.09959555334515041
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,6144,0.08563999997244941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,3584,0.023065777288542852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,5120,0.07952266931533813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,4096,0.06112888786527845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,3072,0.02058577703105079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,2560,0.019744889603720773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,2048,0.016688000824716356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,3584,0.055312001042895846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,1536,0.013495999905798169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,3072,0.04961155520545112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,1024,0.01000711073478063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,2560,0.043587555487950645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,2048,0.0395448903242747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,768,0.008427555362383524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,1536,0.03198488884501987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,512,0.006552888701359431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,1024,0.02629599968592326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,256,0.004898666507667965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,768,0.02390844457679325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,128,0.0038275557259718576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,64,0.002967111145456632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,512,0.02032888929049174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,768,32,0.0028213332924577924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,256,0.017113778326246474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,128,0.015292444162898593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,1024,65536,2.4648630354139542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,64,0.014545778433481852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,32,0.014151111245155334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,16384,0.10509422090318467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,10240,0.03404444456100464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,8192,0.028776887390348647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,12288,0.08764888842900594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,65536,0.2748115592532688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,7168,0.02664533257484436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,12288,0.20052711168924967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,10240,0.14984356032477483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,6144,0.02404444416364034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,5120,0.02086399992307027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,7168,0.09741866588592529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,8192,0.11115554968516032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,16384,0.3292115529378255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,4096,0.01811377704143524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,6144,0.08447021908230251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,3584,0.01722577710946401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,3072,0.017267554998397827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,4096,0.06102666589948866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,2560,0.015695111619101632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,3584,0.05487644341256884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,2048,0.013800889253616333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,3072,0.04929244518280029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,5120,0.07276799943712023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,1536,0.010191111101044549
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,2560,0.04345155424541897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,1024,0.007725333174069722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,2048,0.03752444518937005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,768,0.006515555497672822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,512,0.005375999957323074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,1536,0.03176800078815884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,1024,0.026132444540659588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,256,0.004040889028045866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,768,0.023200889428456623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,128,0.0032666667054096856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,512,0.02010577751530541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,64,0.0026195556339290407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,256,0.016798221402698092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,512,32,0.0024826667375034755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,128,0.01514488955338796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,64,0.014375999569892883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,16384,0.027238221632109746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,32,0.01422755585776435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,12288,0.022255111071798537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,768,65536,2.4675581190321183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,10240,0.019537778364287484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,65536,0.2213768826590644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,8192,0.01700622174474928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,7168,0.0163111107216941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,12288,0.18796800242529976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,8192,0.10976622502009074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,6144,0.016564443707466125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,10240,0.13989155822330052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,16384,0.3145635657840305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,5120,0.014025777578353882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,7168,0.09662755330403645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,4096,0.012002666791280111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,3584,0.011524444652928246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,3072,0.010454222559928894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,6144,0.08476977878146702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,4096,0.061271998617384166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,2560,0.00959644466638565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,3584,0.05527377790874905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,5120,0.07298400004704793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,2048,0.009840000006887648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,3072,0.04933422141604953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,1536,0.008195555872387355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,2560,0.04369599951638115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,1024,0.006360000206364526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,768,0.005570666657553778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,2048,0.03785599933730231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,1536,0.03199733297030131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,512,0.004567111117972268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,1024,0.026320889592170715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,256,0.003588444242874781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,768,0.023152889476882085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,128,0.0029271110478374693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,64,0.0022862222459581164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,256,32,0.002179555594921112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,256,0.016948444975747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,512,0.020280000236299302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,512,65536,2.4950809478759766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,128,0.015472888946533203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,64,0.01463822192615933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,12288,0.016178665889634024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,16384,0.0196088883611891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,32,0.014212444424629211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,10240,0.01645955608950721
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,65536,0.18479555182986787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,8192,0.013735111388895245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,7168,0.012876444392734103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,10240,0.13501066631740993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,12288,0.1826017830106947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,16384,0.31182222896152073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,6144,0.011503111157152386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,8192,0.1096186637878418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,5120,0.010302222437328762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,7168,0.09676000144746567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,3584,0.008275555239783393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,6144,0.08441511127683853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,3072,0.007709333466158972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,5120,0.07254578007592095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,4096,0.008752889103359645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,4096,0.06120355261696709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,2560,0.007057777709431118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,2048,0.006409777949253718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,3584,0.05513777666621738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,1536,0.009104000197516548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,3072,0.049472000863817006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,1024,0.006640000061856375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,2560,0.04331199990378486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,768,0.005548444473081165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,2048,0.03790666659673055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,512,0.004262222184075249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,1024,0.026188444760110643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,1536,0.03194933467441135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,256,0.0031431110368834603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,768,0.02309688925743103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,128,0.002499555548032125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,64,0.0020675555699401433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,512,0.020224889119466145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,128,32,0.0022675556441148124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,128,0.015169777803950839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,256,0.016977777083714802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,64,0.014567999376191033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,32,0.014337778091430664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,16384,0.02555199960867564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,65536,0.14863377147250706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,12288,0.020869332883093093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,256,65536,2.484363555908203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,10240,0.017422222428851657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,8192,0.014689778288205465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,12288,0.17875822385152182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,10240,0.13457332717047796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,7168,0.014249778456158109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,8192,0.1095048917664422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,16384,0.3097013367546929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,6144,0.012729778057999082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,5120,0.010965333216720156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,4096,0.009404444032245213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,3584,0.009121778110663096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,6144,0.08442488643858169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,7168,0.09655644496281941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,3072,0.008373333348168267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,4096,0.06089422437879774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,5120,0.07291644149356417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,3584,0.05512444509400261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,2048,0.006797333144479328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,2560,0.0075288886825243635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,3072,0.04923377765549553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,1536,0.008781332936551835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,1024,0.0064328888224230866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,2560,0.043282667795817055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,768,0.005260444349712796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,2048,0.03779022230042352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,1536,0.031743112537595965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,512,0.0040942223535643685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,1024,0.02625511089960734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,256,0.002938666691382726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,768,0.0229777776532703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,128,0.0023315555105606713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,64,0.0020231112009949153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,64,32,0.0021840000732077495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,256,0.016777777009540133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,512,0.02014755540423923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,128,0.015269334117571512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,64,0.014552889598740472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,16384,0.014671110444598727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,65536,0.09523999691009521
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,32,0.01407911048995124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,12288,0.012177777787049612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,10240,0.010924444430404238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,8192,0.009266667068004608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,128,65536,2.488828447129991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,10240,0.13431733184390598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,12288,0.17760888735453287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,7168,0.010410666465759277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,8192,0.10946489042705959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,16384,0.3084266715579563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,6144,0.007182222273614671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,5120,0.00666222224632899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,4096,0.007393777370452881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,3584,0.006962666908899943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,6144,0.08412355846828884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,7168,0.09675200117958917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,3072,0.0064106666379504735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,5120,0.07286400265163846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,4096,0.06083999739752876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,3584,0.055141333076688975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,2560,0.0058142224119769196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,2048,0.005501333210203383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,1536,0.007629333270920648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,3072,0.04933777782652113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,1024,0.005706666658322017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,768,0.0047288888858424295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,2560,0.043586668041017324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,2048,0.03777777817514207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,512,0.0037742224004533556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,1536,0.03170222375128005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,256,0.002879111096262932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,1024,0.02589599953757392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,128,0.0022524444179402455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,768,0.02292977770169576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,64,0.002187555655837059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,512,0.02008800043000115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,768,32,32,0.002141333288616604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,256,0.016754666964213055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,128,0.015351111690203348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,64,0.014488000008794995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,32,0.014101333088344999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,64,65536,2.490990320841471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,768,32,65536,2.500698725382487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,10240,1.4505075878567164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,12288,1.7632355160183375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,16384,2.525481753879123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,8192,2.414027531941732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,10240,2.932224909464518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,8192,1.1754844453599718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,12288,3.748899671766493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,7168,1.0843529171413846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,6144,0.910919083489312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,7168,2.074405246310764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,5120,0.7880462010701498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,16384,4.566769070095487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,6144,1.8388008541531031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,4096,0.6491911146375868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,5120,1.4792622460259333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,3584,0.6039857864379883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,4096,1.1302293141682942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,3584,0.9727795918782552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,2560,0.46970224380493164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,2048,0.573361767662896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,3072,0.5478560129801432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,2560,0.7033617761400012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,3072,0.8283555772569445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,2048,0.4178648789723714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,1536,0.4655822118123372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,768,0.3024044566684299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,1024,0.30566223462422687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,1536,0.3721911112467448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,768,0.2792417738172743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,256,0.15455733405219182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,512,0.23536978827582467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,512,0.24965598848130968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,1024,0.3697422345479329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,32,0.033048888047536217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,128,0.19532355997297499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,256,0.21520622571309408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,64,0.046783109505971275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,65536,128,0.07454222440719604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,32,0.18284089035458037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,65536,64,0.18318577607472739
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,10240,0.44433066580030656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,10240,0.6779484219021268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,12288,0.5601875517103407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,16384,0.8149911032782661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,16384,1.056873745388455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,12288,0.9593164655897352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,8192,0.3763688935173883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,8192,0.566696908738878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,7168,0.3337848981221517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,6144,0.4278951221042209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,7168,0.48417335086398655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,6144,0.3063875569237603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,5120,0.34986400604248047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,4096,0.2031608952416314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,4096,0.2779093318515354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,5120,0.2560488912794325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,3584,0.2517724566989475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,3072,0.2160631153318617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,2560,0.18116177452935112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,2560,0.14289244016011557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,3584,0.18825777371724448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,3072,0.16939732763502333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,1536,0.09707111120223999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,2048,0.13161067167917886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,768,0.05340533455212911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,1024,0.07280977567036946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,1536,0.10440799925062393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,2048,0.12612799803415933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,512,0.03860355416933695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,1024,0.09027377764383952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,768,0.08304799927605523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,256,0.0217848883734809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,64,0.013245333400037555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,512,0.07220177517996894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,128,0.0144533332851198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,256,0.0646266672346327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,64,0.05468888746367561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,128,0.057824002371893994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,32,0.010464000205198923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,32,0.05420800050099691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,12288,0.4380951192643907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,16384,0.6025688913133409
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,12288,0.6259866820441352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,16384,0.799048900604248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,16384,65536,3.506024890475803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,8192,0.39066576957702637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,10240,0.4921831025017633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,10240,0.37043200598822695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,7168,0.44992531670464414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,8192,0.3009377850426568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,6144,0.4066675504048665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,6144,0.21410755316416422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,5120,0.1949066718419393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,5120,0.3291964530944824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,4096,0.14596977498796251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,3584,0.23044000731574166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,3584,0.13384532928466797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,7168,0.2685306602054172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,4096,0.2783599959479438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,3072,0.19461333751678467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,3072,0.11939644813537598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,2560,0.11446577972835964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,2048,0.12597421805063883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,2560,0.1650142272313436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,2048,0.09467377927568223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,16384,65536,5.433949364556207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,1536,0.10213866498735215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,1024,0.059862222936418324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,1536,0.08299377891752455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,768,0.04634666774008009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,1024,0.06963821914460924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,512,0.029358221424950495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,256,0.018255111243989732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,768,0.06466399961047702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,128,0.012552888856993781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,64,0.010440000229411654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,256,0.05053155620892843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,32,0.008719111482302347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,512,0.05761866437064277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,65536,2.944917254977756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,32,0.04313244422276815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,64,0.04299999939070808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,12288,128,0.04695644312434726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,16384,0.5792880058288574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,12288,0.5797004699707031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,12288,0.4422995514339871
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,12288,65536,4.011379665798612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,16384,0.7931040128072103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,10240,0.4604658020867242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,8192,0.2866986592610677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,10240,0.365140438079834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,7168,0.35462223158942324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,8192,0.3802177906036377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,7168,0.2466097672780355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,6144,0.2650382253858778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,5120,0.2845573425292969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,6144,0.2085111141204834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,5120,0.18308888541327584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,4096,0.14631822374131945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,3072,0.14015555381774902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,3584,0.1544248925315009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,4096,0.1838222212261624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,3584,0.17402933703528511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,2560,0.15665245056152344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,3072,0.12034044000837539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,2048,0.1415733363893297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,2560,0.10396711031595866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,1536,0.09274311198128594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,1024,0.07265777720345391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,768,0.040947556495666504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,2048,0.09230933586756389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,512,0.02880355715751648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,1024,0.0683271090189616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,1536,0.07776355743408203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,128,0.01277955538696713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,768,0.06136800183190239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,256,0.018223999275101554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,512,0.053020444181230336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,65536,2.710177739461263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,32,0.008284444610277811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,10240,64,0.010399999717871347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,128,0.042456888490253024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,256,0.04698044392797682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,32,0.040172444449530706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,64,0.041238221857282854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,10240,65536,2.9201857248942056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,16384,0.5400080151028103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,12288,0.3826248910692003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,10240,0.34876177046034074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,12288,0.46268357170952695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,16384,0.5096577538384331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,8192,0.28059289190504283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,7168,0.25233067406548393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,6144,0.20925599998897979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,10240,0.3546373314327664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,8192,0.2731413311428494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,7168,0.24244711134168836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,6144,0.2058160040113661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,5120,0.1657680008146498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,4096,0.13149155510796443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,5120,0.16946400536431205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,3584,0.11042844586902195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,4096,0.1406951083077325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,2560,0.07750399907430013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,3072,0.110770669248369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,2560,0.11257955763075088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,2048,0.06078400214513143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,3072,0.0952577789624532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,3584,0.12430400318569607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,1024,0.03330488999684652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,768,0.026544888814290363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,1536,0.0506879985332489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,1536,0.07163911395602755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,512,0.020397333635224234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,256,0.01292711082432005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,1024,0.0583768884340922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,128,0.008656889200210571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,2048,0.07955111397637261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,512,0.04993155598640442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,768,0.05210577779346042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,64,0.0069066666894488865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,32,0.005984888722499211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,256,0.04316711094644335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,128,0.03750844465361701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,64,0.03672444489267137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,32,0.03805689016977946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,8192,65536,2.3436817593044705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,12288,0.3935493363274469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,8192,65536,2.44033071729872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,16384,0.5059537887573242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,16384,0.4796115557352702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,10240,0.3223777876959907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,8192,0.2589351071251763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,10240,0.3100399971008301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,12288,0.35630400975545246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,8192,0.2558346589406331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,7168,0.21577066845364043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,6144,0.181750218073527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,5120,0.13988978332943386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,7168,0.20882577366299102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,4096,0.11496267053816055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,3584,0.09969422552320693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,6144,0.18446755409240723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,4096,0.11636000209384495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,5120,0.15303911103142634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,3584,0.10569244623184204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,2560,0.07618311378690931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,65536,1.9512916141086156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,3072,0.08721333079867893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,2048,0.06345511145061918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,3072,0.0958168903986613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,2560,0.09412799941168891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,1536,0.05516711208555433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,1024,0.033760001262029014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,768,0.0251217782497406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,2048,0.06546133094363742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,512,0.019304000669055514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,1024,0.04793955551253425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,1536,0.0551511115498013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,256,0.012368000215954252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,128,0.008170666793982187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,256,0.032624890406926475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,512,0.03931466738382975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,64,0.006623111251327727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,7168,32,0.005922666854328579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,768,0.04561511013242933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,64,0.02923911147647434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,65536,2.274719026353624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,128,0.03051466743151347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,7168,32,0.02888266576661004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,12288,0.298492431640625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,10240,0.31664978133307564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,12288,0.32887999216715497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,16384,0.4208559989929199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,16384,0.45396799511379665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,10240,0.27877243359883624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,8192,0.21537333064609102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,6144,0.16475999355316162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,7168,0.19801333215501574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,8192,0.26426222589280873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,5120,0.13886577553219265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,4096,0.11184444692399766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,7168,0.19217422273423937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,6144,0.15225333637661403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,5120,0.12771822346581355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,3584,0.09877866506576538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,4096,0.10180622339248657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,2560,0.07250044743220012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,2048,0.058635552724202476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,3072,0.08753955364227295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,3072,0.08612800306744045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,1536,0.04536799920929802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,1024,0.03413777881198459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,65536,1.6160915162828233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,768,0.02737066646416982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,3584,0.08957155545552571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,1536,0.052054223087098866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,2560,0.07671911186642118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,512,0.0192266669538286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,1024,0.04368889000680712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,2048,0.060508443249596484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,256,0.012069332930776807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,768,0.045296000109778516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,64,0.006227555374304454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,512,0.034684442811542086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,256,0.030119998587502375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,128,0.0284826656182607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,64,0.026937777797381084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,32,0.026820444398456152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,32,0.005789333333571752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,6144,128,0.008120888637171851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,16384,0.3475004567040338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,12288,0.25373156865437824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,6144,65536,2.2471947140163846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,10240,0.22051466835869682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,16384,0.4213306638929579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,8192,0.16719022062089708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,10240,0.26092176967196995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,12288,0.3216799894968669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,8192,0.1961831119325426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,6144,0.14825599723392063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,7168,0.17272799544864229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,6144,0.13628710640801325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,7168,0.16158400641547307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,5120,0.1385200023651123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,4096,0.10606844557656182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,5120,0.11649955643547906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,3584,0.08872089121076797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,2560,0.06985155741373698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,3584,0.08425955639945136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,3072,0.08143911096784803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,3072,0.07808533642027113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,4096,0.0938897795147366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,65536,1.5385013156467016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,2048,0.08254044585757785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,1536,0.04887288808822632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,768,0.027042667071024578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,2048,0.05871555540296766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,2560,0.06867822011311848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,1024,0.04224977890650431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,512,0.019167999426523846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,1536,0.05048355460166931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,256,0.011784888803958893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,1024,0.03556177682346768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,128,0.008051555189821456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,512,0.03345244460635715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,768,0.03815111186769273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,256,0.0292871097723643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,64,0.006464889066086874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,5120,32,0.004902222090297275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,128,0.027406222290462915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,64,0.02571733295917511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,32,0.025777777036031086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,5120,65536,2.2127511766221786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,16384,0.25759466489156085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,12288,0.20392266909281412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,12288,0.23005599445766875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,16384,0.3050311141543918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,8192,0.1433466672897339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,10240,0.17429599497053358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,10240,0.18515643808576795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,8192,0.1263093286090427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,7168,0.10762400097317165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,7168,0.1121893326441447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,6144,0.09008000294367473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,5120,0.07660000191794501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,4096,0.07060088713963826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,6144,0.08706844515270656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,5120,0.0747306678030226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,3584,0.05254222287072075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,4096,0.06235111422008938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,3584,0.059676441881391734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,3072,0.045511109961403735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,65536,1.148222181532118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,2048,0.03217511044608222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,1536,0.030641777647866145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,3072,0.05274755424923367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,2560,0.047340446048312716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,2560,0.03949866692225138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,1024,0.02150933278931512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,2048,0.04154310954941644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,768,0.015775110986497667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,1536,0.03779200050565932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,256,0.0075137780772315125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,768,0.029156443145540025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,128,0.005463111317820019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,512,0.025984888275464375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,512,0.012443555725945367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,1024,0.03227555420663621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,64,0.004543999830881755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,128,0.02093066606256697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,4096,32,0.004234666625658671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,256,0.023723555935753718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,32,0.02015466656949785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,64,0.01979377700222863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,10240,0.13581866688198513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,16384,0.24694667922125924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,12288,0.19079111682044136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,16384,0.28624267048305935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,4096,65536,1.8650559319390192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,7168,0.10421422455045913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,8192,0.11822399828169082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,10240,0.1651217805014716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,12288,0.20856532785627577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,6144,0.08448266983032227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,8192,0.11707644992404515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,5120,0.07192889187071058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,6144,0.08674755361345078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,4096,0.06914222240447998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,7168,0.10025778081681992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,5120,0.07243377632564969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,3584,0.052390221092436046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,3072,0.0463066664006975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,2560,0.039552887280782066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,65536,1.123636457655165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,3584,0.05681866407394409
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,2048,0.03545422355333964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,3072,0.05068888929155138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,1536,0.02883466747072008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,1024,0.018602665927675035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,2560,0.04553777641720242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,768,0.016167110866970487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,2048,0.040848887628979154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,1536,0.03613866700066461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,4096,0.06084622277153862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,1024,0.030293332205878362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,256,0.007360888852013483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,128,0.005428444594144821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,512,0.025360888904995386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,512,0.011919111013412476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,768,0.027823110421498615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,128,0.02028888960679372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,64,0.004476444588767158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,256,0.02204177776972453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3584,32,0.003976888954639435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,64,0.019648889700571697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,32,0.01868355605337355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,12288,0.15923199388715956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,16384,0.21270133389366996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3584,65536,1.8138951195610895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,12288,0.19066754976908365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,16384,0.27976266543070477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,10240,0.12797155645158556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,8192,0.11329955524868435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,7168,0.0989982220861647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,8192,0.10420355531904434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,7168,0.09165600273344253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,6144,0.08425688743591309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,10240,0.14889600541856554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,5120,0.07166133324305217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,65536,0.8650995360480415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,6144,0.08049955632951525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,4096,0.05761866437064277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,3584,0.05100533366203308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,3072,0.04522044459978739
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,4096,0.05873777468999227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,2560,0.044176889790429004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,2048,0.03270666797955831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,2560,0.03784888982772827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,3584,0.060319999853769936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,5120,0.06976888577143352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,1536,0.02604888876279195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,1024,0.019312888383865356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,2048,0.03881599836879306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,768,0.014716444744004143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,1536,0.03416533271471659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,1024,0.029622223642137315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,512,0.010925333533022137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,256,0.007157333195209503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,128,0.0052844443255000645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,768,0.02770400047302246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,512,0.02348622183005015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,64,0.004361777669853634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,256,0.021174222230911255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,3072,0.04925688770082262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,3072,32,0.0037662221325768363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,64,0.018983110785484314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,32,0.018145778112941317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,65536,1.7768239974975586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,16384,0.16220445103115505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,12288,0.13950933350457087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,3072,128,0.018976888722843595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,8192,0.08625333176718818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,16384,0.2591466638776991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,12288,0.17490043905046251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,10240,0.13419022825029162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,7168,0.09949866930643718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,8192,0.10188533200158013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,10240,0.10983822080824111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,6144,0.08962755733066136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,7168,0.0895093348291185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,5120,0.07972355683644612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,4096,0.05692088603973389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,65536,0.7309057977464465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,3584,0.05058311091528999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,4096,0.05838577614890205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,3072,0.041020443042119346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,5120,0.06838844219843547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,2560,0.03731911049948798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,3584,0.05436977744102478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,6144,0.07856088876724243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,3072,0.05327733357747396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,1536,0.026025777061780293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,2048,0.032617777585983276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,1024,0.018024888303544786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,768,0.014708444476127625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,2560,0.04338666796684265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,512,0.010793777803579966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,1536,0.03374933203061422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,256,0.007116444408893585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,2048,0.03820977939499749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,128,0.005255110975768831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,64,0.004303110970391167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,512,0.02329066726896498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,768,0.026180444492234126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,1024,0.030115554730097454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,256,0.02000977761215634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2560,32,0.003253333270549774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,128,0.01822933389080895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,64,0.017613333132531907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,32,0.01805688937505086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,12288,0.10107733143700494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,16384,0.12618577480316162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,10240,0.08889333407084148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,12288,0.16461955176459417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,8192,0.07651999923917982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,16384,0.24221955405341256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,10240,0.12399110529157852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,7168,0.07502044571770562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,65536,0.6000373098585341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,6144,0.05238666799333361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,8192,0.0983840028444926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2560,65536,1.7568995157877605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,5120,0.049696889188554555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,7168,0.08683466911315918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,4096,0.03771377934349908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,6144,0.07739910814497206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,3584,0.035367111365000405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,3072,0.032074666685528226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,3584,0.051552888419893056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,3072,0.04687022169431051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,2560,0.02826933397187127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,5120,0.06671288940641615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,2560,0.04195911023351881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,4096,0.056603557533688016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,2048,0.02148977749877506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,1024,0.012505778008037143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,1536,0.017678222722477384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,768,0.010201777848932479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,2048,0.03657066822052002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,1536,0.03179288903872172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,512,0.007679999702506595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,1024,0.02645955483118693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,256,0.005324444423119227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,768,0.02438933319515652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,128,0.004287111262480418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,64,0.003551111039188173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,2048,32,0.002965333354141977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,512,0.021199999584092036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,256,0.01867377758026123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,128,0.016612443659040663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,32,0.015804444750150044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,64,0.016197333733240765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,16384,0.0970568921830919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,12288,0.08231111367543538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,10240,0.06533066431681316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,16384,0.2256986565060086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,8192,0.05338577760590447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,12288,0.1476728916168213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,10240,0.12023022439744736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,65536,0.48282755745781797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,2048,65536,1.7478755315144856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,6144,0.04423377911249796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,8192,0.09634488821029663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,5120,0.03746666510899862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,6144,0.07732799980375502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,7168,0.08612355258729722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,7168,0.0481279989083608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,5120,0.0654088921017117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,3584,0.030561778280470107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,3072,0.026624000734753076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,4096,0.03364177876048618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,2048,0.017808000246683758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,3584,0.050867554214265615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,3072,0.045534223318099976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,1536,0.014578666951921252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,4096,0.055699553754594594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,2560,0.040305776728524104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,1024,0.0105591110057301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,2048,0.035325331820382014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,768,0.0090506664580769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,512,0.006832888970772426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,2560,0.022491556074884202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,1024,0.02550044490231408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,1536,0.03052177694108751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,256,0.005010666532648934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,768,0.023225777679019507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,64,0.0033822223130199644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,32,0.0030951110853089225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,512,0.020005333754751418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1536,128,0.003964444415436851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,256,0.017508443858888414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,128,0.01581333412064446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,64,0.01497688889503479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,32,0.014800000521871777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,16384,0.06420355372958712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,12288,0.052904887331856616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,10240,0.044799112611346774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,8192,0.03482222225930955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,16384,0.21473422315385607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,12288,0.14023999373118082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,7168,0.03023911184734768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,65536,0.35591912269592285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,8192,0.09518222014109294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,6144,0.027423999375767175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,10240,0.11946845054626465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,4096,0.02040799955526988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,5120,0.024829333027203877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1536,65536,1.7443129221598308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,7168,0.08511910835901897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,3584,0.018745778335465323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,5120,0.06486311223771837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,4096,0.05468177795410156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,6144,0.07507377862930298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,3584,0.050965332322650485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,3072,0.0170133329100079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,2560,0.015343111422326831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,2048,0.01568533314598931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,1536,0.0123848890264829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,3072,0.04474577638838026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,1024,0.009019555317031013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,768,0.007479999628331926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,2560,0.03971822063128153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,2048,0.03439199924468994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,1536,0.029787556992636785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,512,0.005995555470387141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,1024,0.024831111232439678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,128,0.003604444364706675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,768,0.022092445029152766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,64,0.0029279999434947968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,256,0.004464000049564573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,512,0.019155555301242404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,256,0.016736888223224215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,1024,32,0.002687999978661537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,128,0.01516533394654592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,64,0.014307555225160388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,32,0.014058666096793281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,16384,0.10077510939704047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,12288,0.0414284434583452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,10240,0.03474666674931844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,65536,0.2764186594221327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,16384,0.20498666498396131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,12288,0.13932622803582087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,8192,0.028734223710166082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,7168,0.026721777187453374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,10240,0.11743111080593532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,6144,0.02423200011253357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,8192,0.09503200319078232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,5120,0.021017778250906203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,7168,0.0844951139556037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,6144,0.07496355639563666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,5120,0.0646159980032179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,3584,0.0173333336909612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,4096,0.05433600147565206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,4096,0.01815022197034624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,3072,0.017332444588343304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,1024,65536,1.7557537290785048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,2560,0.01568000018596649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,2048,0.01367644468943278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,3584,0.049788445234298706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,1536,0.010638222098350525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,3072,0.04450577828619215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,1024,0.007818666597207388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,2560,0.039677331844965615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,768,0.006514666808976068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,512,0.005405333307054307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,1536,0.02958933181232876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,1024,0.024316444993019104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,256,0.004018666843573253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,768,0.021942221456103857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,128,0.0032924444725116095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,2048,0.034330666065216064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,64,0.0027315556589100095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,512,0.018998222218619455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,256,0.016380444169044495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,768,32,0.002487999904486868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,128,0.015040889382362366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,16384,0.036524444818496704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,32,0.014031999640994601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,64,0.014209777116775513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,12288,0.027615999182065327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,10240,0.024010666542583044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,65536,0.22167555491129556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,8192,0.02027555637889438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,7168,0.018989332848125033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,10240,0.11595288912455241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,12288,0.13852889007992214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,16384,0.19868355327182344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,6144,0.017272889614105225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,5120,0.0169831116994222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,4096,0.01457244488928053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,7168,0.08470400174458821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,8192,0.09451555543475681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,3584,0.01367200000418557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,768,65536,1.7623679902818468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,5120,0.065010666847229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,3072,0.012359110845459832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,4096,0.054661333560943604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,2048,0.009871111147933537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,6144,0.074654221534729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,2560,0.011206222077210745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,3584,0.049498667319615684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,1536,0.008303999900817871
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,3072,0.04455644554562039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,1024,0.006350222147173352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,768,0.00554044461912579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,2560,0.03973155551486545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,512,0.004544888933499654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,256,0.003735110991530948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,2048,0.03457244568400913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,1024,0.0244168887535731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,1536,0.029587556918462116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,768,0.02187999917401208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,128,0.003079111170437601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,512,0.018976888722843595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,64,0.002512000087234709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,512,32,0.0023093333260880574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,256,0.01640266676743825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,128,0.01497511068979899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,16384,0.02348888913790385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,64,0.01424533294306861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,32,0.01407199932469262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,12288,0.019157333506478202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,10240,0.016799110505315993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,65536,0.13780089219411215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,8192,0.014399111270904541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,7168,0.013162666724787818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,12288,0.13630755742390951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,16384,0.1854791111416287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,10240,0.11528533034854466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,8192,0.09476088815265232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,6144,0.012160888976520963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,5120,0.011917333636018964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,7168,0.0846995578871833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,6144,0.07487733496559991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,5120,0.06474133332570393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,3584,0.00963288876745436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,4096,0.010208889014191097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,4096,0.0546231104267968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,2560,0.007932444413503012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,2048,0.006968888971540663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,512,65536,1.7677218119303386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,3584,0.04969955484072367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,3072,0.044658667511410184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,2560,0.039613333013322614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,1024,0.006779555645253923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,3072,0.008767999708652496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,1536,0.00923911150958803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,2048,0.034338666333092585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,512,0.004393777913517422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,1536,0.029694222741656836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,1024,0.024578667349285547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,256,0.0032479998966058097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,768,0.02200177808602651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,128,0.0026195556339290407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,768,0.005647111270162795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,64,0.0021386666016446217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,512,0.01906399925549825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,256,0.016544888416926067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,256,32,0.00231200001306004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,128,0.015164444843928019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,16384,0.016343111793200176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,65536,0.055247111452950366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,64,0.014463999205165438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,12288,0.013604444762070974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,32,0.014184888866212634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,10240,0.01202488856183158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,8192,0.010397333237859938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,7168,0.009661333428488838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,16384,0.1829733318752713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,12288,0.1362124416563246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,8192,0.09484355317221747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,10240,0.11494933234320746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,6144,0.008959111240175035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,5120,0.00794044468137953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,7168,0.08477422263887192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,4096,0.007024889190991719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,6144,0.07453244262271456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,3584,0.00685066646999783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,5120,0.06459466616312663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,3072,0.006315555423498154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,256,65536,1.7704284456041124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,4096,0.05460177858670553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,2560,0.005938666562239329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,3584,0.04968800147374471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,1536,0.00885333369175593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,3072,0.04466933343145582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,2560,0.03930844532118903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,1024,0.006508444332414203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,2048,0.034479111433029175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,768,0.0053582220441765254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,1536,0.029578665892283123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,512,0.00416355538699362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,2048,0.005492444253630108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,256,0.0029893333299292457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,128,0.0023973333752817577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,1024,0.02438844409253862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,768,0.02180800007449256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,64,0.002011555557449659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,512,0.019002666076024372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,128,32,0.0021964444054497615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,256,0.016520000166363187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,128,0.015083556373914083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,16384,0.017249777913093567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,64,0.014409777190950183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,32,0.013984888792037964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,65536,0.05747555361853706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,12288,0.014595554934607612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,10240,0.012888000243239932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,8192,0.007298666569921706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,7168,0.0070017774899800616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,16384,0.1804293394088745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,8192,0.09470399883058335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,12288,0.13602399826049805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,10240,0.11472266250186497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,6144,0.009207111265924241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,5120,0.008526222573386298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,7168,0.08477599753273858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,4096,0.007381333245171442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,3584,0.007033777733643849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,6144,0.07472088601854113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,5120,0.06442222330305311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,3072,0.007002666592597961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,4096,0.05443466703097025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,3584,0.0495324432849884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,2560,0.0063866666621632045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,2048,0.005710222240951326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,1536,0.00518844442235099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,3072,0.044601778189341225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,2560,0.03945066531499227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,1024,0.005758222192525864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,128,65536,1.7770889070298936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,2048,0.034345779154035784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,768,0.004829333474238713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,1536,0.02940000096956889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,256,0.0029120000286234748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,512,0.004027555386225383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,1024,0.024311999479929607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,128,0.0022524444179402455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,768,0.021839999490314062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,64,0.002313777804374695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,512,0.018968888454967074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,64,32,0.002151111140847206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,256,0.01629244453377194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,128,0.015054222610261707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,65536,0.031018667750888403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,64,0.014135110709402295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,16384,0.010862222148312463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,12288,0.00925866680012809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,32,0.014124444789356656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,10240,0.007091555330488417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,8192,0.006515555497672822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,7168,0.0075635554061995606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,16384,0.18213510513305664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,10240,0.11497244569990371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,12288,0.13553422027164036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,8192,0.09463200304243301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,6144,0.006966222491529252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,7168,0.08466755681567723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,5120,0.0063582224150498705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,4096,0.005702222386995952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,3584,0.0058497778243488735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,3072,0.005370666583379109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,6144,0.07464710871378581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,5120,0.06457600328657362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,4096,0.05449066559473673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,2560,0.005554666535721884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,3584,0.04945066571235657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,2048,0.004658666749795278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,1536,0.004260444392760594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,3072,0.04433244466781616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,1024,0.005601777798599667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,2560,0.039304888910717435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,2048,0.034308443466822304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,768,0.004648000001907349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,1536,0.02943199872970581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,512,0.003657777690225177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,64,65536,1.7760719723171656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,256,0.0027706666539112725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,1024,0.024115555816226538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,64,0.002151111140847206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,128,0.0022880000372727713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,768,0.02164622147878011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,512,0.01891377733813392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,256,0.016339555382728577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,512,32,32,0.0021395554973019496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,128,0.015000889698664347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,64,0.014189332723617554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,32,0.01401600076092614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,512,32,65536,1.772441758049859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,10240,1.3778151406182184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,12288,1.7385395897759333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,8192,1.0843547185262044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,16384,2.404237323337131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,8192,2.0743484497070312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,10240,2.5276506212022567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,7168,0.9374826219346789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,12288,3.2646827697753906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,7168,1.781826655069987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,5120,0.6934977637396919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,6144,0.8720986578199598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,6144,1.5975866317749023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,5120,1.3163697984483507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,16384,3.9429651896158853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,4096,0.578479978773329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,4096,1.0243564181857638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,3072,0.4705377684699164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,3584,0.5475102000766331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,3584,0.8784009085761176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,2560,0.41115111774868435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,3072,0.7583466635810004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,2560,0.6463982264200846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,1536,0.4222302171919081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,2048,0.5645990901523167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,2048,0.36086755328708225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,1024,0.22869867748684355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,1536,0.2952631049686008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,512,0.20087288485633004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,1024,0.3327111138237847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,512,0.18261155817243788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,256,0.11508799923790826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,768,0.2659040027194553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,128,0.05537688732147217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,64,0.029559112257427637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,256,0.16286577118767634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,768,0.20781777964697945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,65536,32,0.024654222859276667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,64,0.1390088929070367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,128,0.15459822283850774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,65536,32,0.13952088356018066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,12288,0.8054009013705783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,12288,0.541520012749566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,16384,0.7613306575351291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,10240,0.6696960131327311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,16384,1.0774941974216037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,10240,0.4114728768666585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,8192,0.5166648758782281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,8192,0.33868177731831867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,7168,0.4929724269443088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,7168,0.2997173203362359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,5120,0.21369066503312853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,6144,0.4348417652977838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,6144,0.2585413455963135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,4096,0.29411376847161186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,5120,0.37561866972181535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,4096,0.16186932722727457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,3584,0.156333327293396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,3072,0.2204160028033786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,3584,0.2663244406382243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,2560,0.18343288368648955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,2560,0.11489421791500515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,2048,0.13877777258555093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,1024,0.06275555822584364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,1536,0.11118133862813313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,3072,0.14181244373321533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,768,0.04978400137689379
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,1024,0.07105866405698988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,256,0.01752088963985443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,512,0.03460177779197693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,2048,0.10360088613298203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,1536,0.08798488643434312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,768,0.06384533643722534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,512,0.0569288863076104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,128,0.012674666941165924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,64,0.01033155537313885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,65536,2.903708351982964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,32,0.008685333033402761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,256,0.05011377731959025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,128,0.045964443021350436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,32,0.04344711038801405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,16384,64,0.043567111094792686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,16384,65536,3.7684241400824656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,16384,0.6396515634324815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,12288,0.4783466657002767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,12288,0.7134088940090604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,16384,0.8795484436882867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,10240,0.5733057657877604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,8192,0.2980275683932834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,8192,0.42539021703932023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,7168,0.2641315460205078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,7168,0.37203023168775773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,10240,0.40077511469523114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,6144,0.34807999928792316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,5120,0.27759912278917104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,4096,0.22395110130310059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,5120,0.1841519938574897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,6144,0.23877154456244576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,4096,0.15522578027513292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,3584,0.20175911320580375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,3072,0.16495911280314127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,2560,0.13335822688208684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,3584,0.13949155807495117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,2048,0.10672355360454983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,2560,0.10906489027871026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,1536,0.08221066660351224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,3072,0.12397866778903538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,1536,0.0771626697646247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,768,0.0469777782758077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,512,0.027744000156720478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,1024,0.06209511227077908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,256,0.016754666964213055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,768,0.05558844407399496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,1024,0.05996711386574639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,128,0.011591110792424945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,256,0.043028444051742554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,64,0.009692444569534725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,512,0.05010133319430881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,2048,0.09485777881410386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,32,0.007864000068770515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,64,0.03635644581582811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,128,0.0391937792301178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,32,0.03530044356981913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,12288,65536,2.803014119466146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,12288,65536,3.044791115654839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,16384,0.4713128937615289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,12288,0.5065546565585667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,16384,0.6807946628994412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,10240,0.4032417668236627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,8192,0.3182986577351888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,10240,0.27936887741088867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,8192,0.2232711050245497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,12288,0.3728462325202094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,7168,0.1921422216627333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,7168,0.290084441502889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,6144,0.23978845278422037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,5120,0.21180621782938638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,5120,0.1298711167441474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,4096,0.10769510931438869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,6144,0.18617510795593262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,4096,0.16098399957021078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,3584,0.13345332940419516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,3072,0.1046159995926751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,2560,0.09940889146592881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,3584,0.10379199849234687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,3072,0.09069599707921346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,2048,0.07964444160461426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,1536,0.05475199884838528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,1024,0.037880887587865196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,2048,0.0709644423590766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,768,0.025592888394991558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,1024,0.049385776122411094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,65536,2.1907439761691623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,512,0.018581334087583754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,2560,0.08139110936058892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,1536,0.06060710880491468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,128,0.008428444465001425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,256,0.012888000243239932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,64,0.007040888898902469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,65536,2.498689863416884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,10240,32,0.00573600000805325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,256,0.03664711117744446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,512,0.04071733355522156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,128,0.03282311227586534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,768,0.045356445842319064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,32,0.030930668115615845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,10240,64,0.031031999323103163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,16384,0.49015288882785374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,12288,0.3772764470842149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,16384,0.4630097813076443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,10240,0.3222444322374132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,12288,0.38930310143364805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,8192,0.2426097657945421
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,10240,0.32258934444851345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,7168,0.21667822202046713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,6144,0.2118799951341417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,7168,0.2427297698126899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,8192,0.25527911716037327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,6144,0.19889511002434623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,5120,0.15235822730594212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,3584,0.10020799769295587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,5120,0.15312443839179143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,4096,0.1403884490331014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,4096,0.11537867122226292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,3584,0.12535911136203343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,2048,0.07090044683880277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,2560,0.08754399749967788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,3072,0.08695022265116374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,3072,0.09970577557881673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,2560,0.08678222364849514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,1536,0.05546489026811388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,1024,0.03201955556869507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,768,0.0269777774810791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,512,0.019539554913838703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,1536,0.0632257792684767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,2048,0.07481066385904948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,256,0.012098666694429187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,128,0.00815466625822915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,512,0.04318133327696058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,64,0.006340444501903322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,256,0.03684711125161912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,768,0.046147555112838745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,1024,0.05411466625001696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,32,0.00544177782204416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,128,0.036879112323125206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,8192,65536,1.9370969136555989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,64,0.03400177756945292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,32,0.03224533465173509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,8192,65536,1.9590596093071833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,12288,0.3284204536014133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,10240,0.2555777761671278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,16384,0.4203182326422797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,12288,0.3292062282562256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,16384,0.43355290095011395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,7168,0.19251644611358643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,8192,0.22328445646497938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,6144,0.1696444484922621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,7168,0.18569778071509468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,10240,0.2882568836212158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,6144,0.17197155952453613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,4096,0.10913778013653225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,8192,0.2311555544535319
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,5120,0.1436577770445082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,3072,0.10492799679438274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,5120,0.13052266173892552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,4096,0.12297333611382379
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,3584,0.10433155298233032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,3584,0.107424000898997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,2560,0.09188355339898004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,3072,0.083588441212972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,2048,0.07917333311504789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,1536,0.055198222398757935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,2560,0.0779786639743381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,1024,0.031167109807332356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,2048,0.06402133570777045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,1536,0.05361510978804695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,768,0.024319999747806128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,65536,1.7666311264038086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,768,0.040793776512145996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,256,0.011954666839705573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,1024,0.0451200008392334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,512,0.019430221782790292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,512,0.036284443404939436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,64,0.006115555349323485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,128,0.008009778128729926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,7168,32,0.005011555635266834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,256,0.03377422359254625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,128,0.030618667602539062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,65536,1.8992898729112413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,64,0.028875556257035997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,7168,32,0.028594666057162817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,10240,0.23164712058173287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,16384,0.38115111986796063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,12288,0.2892755667368571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,16384,0.4102151128980849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,12288,0.315008004506429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,7168,0.17226044336954752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,10240,0.2491768995920817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,8192,0.20364533530341256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,7168,0.16640355851915148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,8192,0.2000044451819526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,6144,0.15489066971672907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,6144,0.14488977856106228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,5120,0.14212444093492296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,5120,0.11493155691358779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,3584,0.09324977795283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,4096,0.10782755745781793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,3584,0.08963200118806626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,65536,1.4891778098212347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,2560,0.07402222024069892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,3072,0.0865946676996019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,4096,0.10267733203040229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,2048,0.08247022496329413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,3072,0.07426044676038954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,1536,0.05214133196406894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,1024,0.03528533379236857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,1536,0.05106755428844028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,768,0.026874666412671406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,512,0.019208000765906442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,2560,0.06710133287641737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,256,0.011787555283970304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,1024,0.04186044467820061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,2048,0.058210664325290255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,128,0.008070222205585903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,64,0.005995555470387141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,65536,1.8703289031982422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,6144,32,0.0046693334976832075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,512,0.03420622150103251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,256,0.03013955553372701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,128,0.028741333219740126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,64,0.026672000686327618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,768,0.039269334740108915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,6144,32,0.02593155536386702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,12288,0.24124532275729707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,10240,0.2072542243533664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,10240,0.2194382217195299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,12288,0.27724001142713756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,16384,0.2969013320075141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,8192,0.15199377801683214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,16384,0.3813955518934462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,6144,0.10498399866951837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,8192,0.17015022701687285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,7168,0.11989777617984348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,5120,0.08676177925533718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,7168,0.14172800381978354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,3584,0.061390221118927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,4096,0.07185777690675524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,5120,0.10524444447623359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,3072,0.05355377660857307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,4096,0.0904044442706638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,3584,0.08173866404427423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,6144,0.12838666968875462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,2560,0.04567555586496989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,2048,0.037568890386157565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,3072,0.07252355416615804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,1536,0.025754666990704004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,1024,0.018741332822375827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,768,0.01513066722287072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,512,0.011179555621412067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,65536,1.2861848407321506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,2048,0.05679821968078613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,2560,0.06468000014623006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,768,0.03688800003793504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,256,0.007816889219813878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,128,0.005502222312821283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,1536,0.0488008889887068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,1024,0.040864888164732195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,64,0.00453955555955569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,5120,32,0.003915555361244413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,512,0.032780445284313627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,256,0.02884711159600152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,64,0.025390222668647766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,32,0.025273778372340735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,128,0.02780710988574558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,16384,0.24247556262546113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,12288,0.19518489307827422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,16384,0.24866578314039442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,5120,65536,1.8368790944417317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,10240,0.15083111657036677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,10240,0.12592266665564641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,12288,0.1848364406161838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,8192,0.12451644738515218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,8192,0.09761599699656169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,6144,0.09562133418189155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,7168,0.08611644638909234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,6144,0.07523377736409505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,5120,0.07295644283294678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,4096,0.06060355239444309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,7168,0.10775555504692926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,65536,1.02583556705051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,3584,0.05173155665397644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,5120,0.06820711162355211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,3072,0.04847200049294365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,3584,0.051242666112052075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,3072,0.049662222464879356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,2560,0.04189599884880913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,2048,0.030976000759336684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,2560,0.042176001601748996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,2048,0.037250667810440063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,1024,0.018017777138286166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,768,0.01477866702609592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,1536,0.03350044290224711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,512,0.010823110739390055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,1024,0.02829244401719835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,1536,0.025264889001846313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,4096,0.05720622009701199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,256,0.007178666690985362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,65536,1.4651538001166449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,128,0.005407111098368962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,512,0.023572444915771484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,256,0.02031822171476152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,768,0.025427555044492085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,32,0.0037946667936113146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,4096,64,0.004394666602214177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,128,0.018559111489189994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,32,0.0188400000333786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,4096,64,0.01867822143766615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,12288,0.17336532804701063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,16384,0.23073956701490614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,16384,0.2364657719930013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,12288,0.15553066465589735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,8192,0.11210400528377956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,10240,0.1349288887447781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,7168,0.08962488836712307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,10240,0.11602666642930771
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,6144,0.07834755712085299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,7168,0.0840266678068373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,5120,0.06758399804433186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,4096,0.05784089035458035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,6144,0.0743537810113695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,5120,0.06439377864201863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,3584,0.051093333297305636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,8192,0.0932426651318868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,65536,0.8946355183919271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,4096,0.05526399943563673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,3072,0.04487999942567614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,2560,0.03803733322355483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,2048,0.035065776771969266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,3072,0.04574577675925361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,1536,0.025935111774338618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,1024,0.019350222415394254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,2560,0.04166400101449754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,2048,0.03682577941152785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,512,0.010863111250930362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,768,0.014615111880832247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,3584,0.05117866728040907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,1024,0.02752444479200575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,512,0.022425777382320825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,1536,0.03247111042340597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,256,0.007181333170996771
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,128,0.005256888767083486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,64,0.004348444441954295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3584,32,0.0033653332955307434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,256,0.020408888657887776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,128,0.018370666437678866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,768,0.025495999389224585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,64,0.01773955590195126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,32,0.018428444862365723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3584,65536,1.4233200285169814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,16384,0.19124800629085967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,12288,0.1639031171798706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,10240,0.11926666895548503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,16384,0.21469510926140678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,12288,0.14034844769371882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,7168,0.09198755688137478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,8192,0.10175289048088922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,8192,0.09233688645892674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,10240,0.1127084493637085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,7168,0.08231288856930204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,6144,0.08127466837565105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,5120,0.0705946683883667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,6144,0.07365777757432726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,65536,0.7921813329060873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,4096,0.054535110791524254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,3584,0.050127999650107495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,5120,0.06417066521114774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,3072,0.04553777641720242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,4096,0.05477422144677904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,3584,0.05053244365586174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,2560,0.04138933287726508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,2560,0.03735911183887058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,1536,0.025811556312772963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,2048,0.034889777501424156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,1024,0.018031999468803406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,3072,0.04556177722083198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,768,0.015152000718646579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,512,0.010656889114114972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,1536,0.03221599923239814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,1024,0.027412444353103638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,256,0.007028444773621029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,2048,0.03640444411171807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,768,0.02511377798186408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,128,0.005180444568395615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,64,0.00424799985355801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,3072,32,0.00313688897424274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,512,0.022704000274340313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,256,0.019880889190567862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,128,0.017981333865059745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,64,0.0177484436167611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,65536,1.4007519616021051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,3072,32,0.017312889297803242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,16384,0.16036178006066218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,12288,0.12788532839881048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,10240,0.0999893347422282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,16384,0.1992124451531304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,8192,0.0804613365067376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,12288,0.1332319974899292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,10240,0.10980088843239678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,8192,0.08973599804772271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,7168,0.06439199712541369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,6144,0.05717244413163927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,5120,0.04906577865282694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,65536,0.6640311347113715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,4096,0.04052177733845181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,3584,0.03492088781462775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,3072,0.030108445220523413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,4096,0.05285244517856174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,6144,0.07145155800713433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,5120,0.062071111467149526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,3584,0.04844089017974006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,7168,0.08078755272759332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,2560,0.02641777859793769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,2048,0.02084000077512529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,1536,0.016774222254753113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,1024,0.012072888513406118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,2560,0.03923111160596212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,768,0.009977777798970541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,3072,0.0440906650490231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,2048,0.03456977672047085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,1536,0.030322667625215318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,1024,0.02531288895342085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,256,0.005296000176005893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,512,0.00741155528359943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,768,0.02295733326011234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,128,0.0041653331783082755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,64,0.003532444437344869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2560,32,0.0029271110478374693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,256,0.018238221605618794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,512,0.02054933375782437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,128,0.016395555602179635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,64,0.01566133399804433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,65536,1.3832755618625219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2560,32,0.015611555841233997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,16384,0.11843289269341363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,12288,0.08845422003004287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,10240,0.07709422376420763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,8192,0.06285244226455688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,16384,0.19263466199239096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,12288,0.12941422727372912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,7168,0.05518933468394809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,10240,0.10788977808422512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,8192,0.08902844455507065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,6144,0.04900266726811727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,5120,0.041767110427220665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,7168,0.08000266551971436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,65536,0.5567910936143663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,4096,0.034167998366885714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,3584,0.030637333790461223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,5120,0.061145775847964816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,3072,0.026449778013759192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,4096,0.05235111051135593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,2560,0.022064889470736187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,6144,0.0707813302675883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,1536,0.014262222581439547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,3584,0.04775822162628174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,2560,0.03835466504096985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,3072,0.0429440008269416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,2048,0.033794667985704206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,2048,0.017663111289342243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,1024,0.010504888991514841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,768,0.00868888861603207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,256,0.005024888863166173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,512,0.0068693334857622785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,1536,0.02936799989806281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,1024,0.024472888973024156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,768,0.02215555475817786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,128,0.00397511116332478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,512,0.019745777050654095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,64,0.0034053333931499054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,2048,32,0.0030702222138643265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,256,0.017103110750516255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,128,0.01564088960488637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,64,0.014877332581414117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,32,0.015186667442321777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,16384,0.08619288603464763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,12288,0.06799644231796265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,10240,0.05863644679387411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,16384,0.173363553153144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,12288,0.12679378191630045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,8192,0.0465848876370324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,65536,0.4369377824995253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,10240,0.1071395542886522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,2048,65536,1.3737768597073023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,7168,0.04193866584036085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,6144,0.037189334630966187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,8192,0.08866844574610393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,7168,0.0790826678276062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,6144,0.06990844673580594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,5120,0.030617776844236586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,5120,0.060679111215803355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,3072,0.020261334048377145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,4096,0.05167466733190748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,4096,0.025361778007613286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,3584,0.0229120006163915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,2560,0.018624888526068795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,2048,0.016791999340057373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,3584,0.046887109676996865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,1536,0.013358222113715278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,3072,0.042316443390316434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,2560,0.03772888912094964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,1024,0.010056888891590966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,2048,0.03303644392225478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,768,0.008429333567619324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,512,0.006615110983451207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,1536,0.028468443287743464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,256,0.004920000003443824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,128,0.00371111101574368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,1024,0.023767110374238756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,768,0.021457778082953557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,64,0.003018666679660479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1536,32,0.0027911110470692315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,256,0.016555555992656283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,512,0.01880088945229848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,128,0.015049777097172208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,64,0.014223110344674854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,32,0.014327110515700446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,16384,0.0988044473859999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,12288,0.04583288894759285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1536,65536,1.3740178214179144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,10240,0.03914755582809448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,16384,0.16697688897450766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,65536,0.30210754606458873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,8192,0.03111999895837572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,12288,0.12575111124250624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,7168,0.027057778504159715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,10240,0.10682133171293472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,6144,0.02425066630045573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,5120,0.02095288866096073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,8192,0.08817599879370795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,7168,0.07890222469965617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,4096,0.01827911039193471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,3584,0.01716977854569753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,6144,0.06978044244978163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,3072,0.01705511079894172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,5120,0.06029422415627373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,4096,0.051289776961008705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,2560,0.015711999601787992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,2048,0.013653332988421122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,1536,0.01034044474363327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,3584,0.04669422242376539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,2560,0.0373324453830719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,768,0.006495999793211619
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,1024,0.007658667034573025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,2048,0.032857778999540545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,3072,0.04195911023351881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,512,0.005395555661784277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,1536,0.028387556473414104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,256,0.004010666575696734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,128,0.0033075554917256036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,64,0.0026622222115596137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,768,0.02103999919361538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,512,0.018446221947669983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,256,0.016100444727473788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,1024,32,0.002451555596457587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,1024,0.023520888553725347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,128,0.014845333165592618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,64,0.01421866648726993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,16384,0.04516444603602091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,32,0.014064888159434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,12288,0.03592888845337762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,10240,0.026386666629049513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,16384,0.16600178347693548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,8192,0.02257333364751604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,12288,0.12611822287241617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,7168,0.020816889074113634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,10240,0.10676799880133735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,1024,65536,1.3887404335869684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,65536,0.24771287706163195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,6144,0.018958222534921434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,8192,0.08811110920376247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,5120,0.01745244363943736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,3584,0.016606221596399944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,6144,0.06967733303705852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,5120,0.06047111087375217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,4096,0.017860444055663217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,7168,0.07896800173653497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,3072,0.015153777268197803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,2560,0.013424889081054263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,4096,0.05134133497873942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,2048,0.0107342220014996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,1536,0.008952889177534316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,3072,0.04198222359021505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,3584,0.04663822385999891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,1024,0.006764444626039929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,2560,0.037603557109832764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,2048,0.03281689021322463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,768,0.005838222387764189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,1536,0.028396444188223943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,512,0.004944889081848992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,256,0.0038728887836138406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,128,0.003119999956753519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,1024,0.02352622151374817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,768,0.021201777789327834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,512,0.018518222702874076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,64,0.0025537777692079544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,768,32,0.0023431111541059283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,256,0.016158221496476066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,128,0.014879110786649915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,64,0.01423911088042789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,16384,0.03176000052028232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,32,0.014081777797804939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,12288,0.021712889273961384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,10240,0.019183110859658983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,65536,0.1932204431957669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,8192,0.01639111174477471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,16384,0.1645573377609253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,12288,0.12534488572014704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,7168,0.01609866652223799
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,768,65536,1.3966764873928492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,10240,0.10681155655119155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,6144,0.016711110870043438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,8192,0.08849066495895386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,4096,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,5120,0.013997332917319404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,7168,0.07904177904129028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,3584,0.011570666399266986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,6144,0.06970577769809298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,5120,0.06058222055435181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,3072,0.010519111322032081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,2560,0.01167555567291048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,2048,0.009862222605281407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,3584,0.046903110212749906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,1536,0.008192888564533657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,3072,0.04210933380656772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,4096,0.051685333251953125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,1024,0.006346666564544042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,2560,0.037646220790015325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,768,0.005597333527273602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,2048,0.03288799855444167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,512,0.004564444637960858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,1024,0.02370044423474206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,256,0.003648888733651903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,1536,0.02845777736769782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,768,0.021198223034540813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,128,0.0029208889851967492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,64,0.0023537776950332853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,512,0.018812444474962022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,512,32,0.002653333254986339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,256,0.01632355484697554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,128,0.014932443698247274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,16384,0.019534221953815885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,64,0.014334221680959066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,32,0.014334221680959066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,12288,0.016252444850073922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,65536,0.0616204473707411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,10240,0.014497778481907315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,8192,0.01256977766752243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,7168,0.01128799964984258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,6144,0.011423110961914062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,12288,0.1252408954832289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,10240,0.10682666964001125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,8192,0.08849155240588719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,16384,0.16306222809685603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,7168,0.07905866702397664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,5120,0.010041777458455827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,4096,0.008760888543393876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,3584,0.008316444853941599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,6144,0.06974222262700398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,512,65536,1.4050328996446397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,3072,0.007753777835104201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,4096,0.05159733361668057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,5120,0.060828447341918945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,2560,0.007137777904669444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,3584,0.04669777883423699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,2048,0.0064151109092765385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,3072,0.04199999901983473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,1536,0.009043555292818282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,2560,0.03752355443106757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,1024,0.006647111227114995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,768,0.005509333478079901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,512,0.004299555387761858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,2048,0.033193777004877724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,1536,0.02833333280351427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,256,0.0031191110610961914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,1024,0.023489778240521748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,128,0.0024684444069862366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,768,0.021200888686709937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,64,0.00208177769349681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,512,0.01870666609870063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,256,32,0.0022604444788561925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,256,0.016415110892719693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,128,0.015023999743991427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,65536,0.04305155409706963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,64,0.01423822177780999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,16384,0.013788444300492605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,32,0.014090667168299357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,12288,0.011559999651379056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,10240,0.010303999814722273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,8192,0.009188444250159794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,7168,0.008329778081840938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,16384,0.1628133323457506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,10240,0.10648266474405925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,8192,0.08845333258310954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,12288,0.12493688530392116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,6144,0.00778311077091429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,5120,0.007073777417341868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,7168,0.07906311088138156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,256,65536,1.404343075222439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,4096,0.006334222025341458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,3584,0.006311111152172089
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,3072,0.005951111101441913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,6144,0.06963733169767591
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,5120,0.060361776087019176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,2560,0.005647111270162795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,4096,0.05158933334880405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,2048,0.005023111071851519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,3584,0.04681244492530823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,1536,0.00508977762526936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,3072,0.04211555586920845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,2560,0.03738311264250014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,1024,0.006430222342411677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,2048,0.03297422329584757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,768,0.0052675555149714155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,512,0.004102222207519743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,1536,0.02825333343611823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,256,0.0029688889367712867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,1024,0.023718222975730896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,128,0.0023208889696333143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,64,0.0019786666250891155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,768,0.021047110358874004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,512,0.018584888842370775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,128,32,0.002185777864522404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,256,0.016193777322769165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,128,0.014944000376595391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,16384,0.014524444937705994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,65536,0.04424622323777941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,64,0.014096000128322177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,32,0.013963555296262106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,12288,0.01221422188811832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,10240,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,8192,0.00925422211488088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,7168,0.009069333473841349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,16384,0.16270221604241267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,12288,0.12520266903771296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,10240,0.10636266734864976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,8192,0.08786133262846206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,6144,0.00812177773978975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,5120,0.007718222008811102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,4096,0.006798222247097228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,128,65536,1.4035786522759333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,7168,0.07895911402172513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,6144,0.06967733303705852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,3584,0.006962666908899943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,3072,0.006421333385838403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,4096,0.051163554191589355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,5120,0.06017866399553087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,2560,0.005864000154866113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,2048,0.005607999861240387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,3584,0.04660711023542616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,1536,0.00489066665371259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,3072,0.04197422332233853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,2560,0.037476443582110934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,1024,0.005657777604129579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,2048,0.03267466690805223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,768,0.0047315553658538395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,1536,0.0280675556924608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,512,0.0037502224246660867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,1024,0.023524444964196947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,256,0.0028008888992998335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,768,0.020967110991477966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,128,0.0022560000005695554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,64,0.0021706666383478376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,512,0.018442667192882962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,64,32,0.0021475555582178966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,256,0.01601777805222405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,128,0.014764444695578681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,16384,0.008051555189821456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,65536,0.02407199972205692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,64,0.014268444644080268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,32,0.014196443888876172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,12288,0.007305777735180325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,8192,0.006238222122192383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,10240,0.006753777878151999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,16384,0.16262133916219076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,7168,0.007043555378913879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,10240,0.10655911101235284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,12288,0.12496177355448405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,5120,0.006011555592219035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,8192,0.08786311414506699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,6144,0.0064497776329517365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,64,65536,1.4046950870090062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,4096,0.005774222314357758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,7168,0.07865155405468412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,6144,0.06970400280422635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,3584,0.005564444594913059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,5120,0.06040088997946846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,3072,0.005153777698675792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,2560,0.0047022220161226065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,2048,0.004618666652176115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,3584,0.04635111159748501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,1536,0.004050666673315896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,4096,0.05145422286457486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,3072,0.04189244574970669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,1024,0.005537777725193236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,2560,0.03737955623202854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,768,0.004568888909286923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,2048,0.03302755620744493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,512,0.0036319999231232535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,1536,0.028022223048739966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,256,0.002741333304180039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,128,0.0022257777551809945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,768,0.020890666378868952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,512,0.01845066746075948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,1024,0.02350577712059021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,64,0.0021377777059872947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,384,32,32,0.0021226666867733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,128,0.014984889162911309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,256,0.01608444419172075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,64,0.01404088901148902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,32,0.014178666803571912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,384,32,65536,1.4076959821912978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,10240,1.458229382832845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,12288,1.7348719702826605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,8192,1.12336974673801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,16384,2.248666763305664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,10240,2.2404585944281683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,8192,1.8398524390326605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,12288,2.928566190931532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,7168,0.9532417721218533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,5120,0.6619635687934028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,6144,0.7960489061143664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,16384,3.5318497551812063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,4096,0.5376319885253906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,7168,1.6491609149509008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,6144,1.419375949435764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,5120,1.2654320398966472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,3584,0.8190053304036459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,3584,0.5113217565748427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,4096,0.9468035168117948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,3072,0.4233226776123047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,2560,0.5924524201287164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,2048,0.5073137813144261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,2560,0.3858124415079753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,3072,0.7624942461649576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,1536,0.40048355526394314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,1024,0.2901502185397678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,1536,0.2606737878587511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,768,0.21551911036173502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,1024,0.1879582272635566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,128,0.03717066513167487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,512,0.1533964474995931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,2048,0.32197867499457467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,256,0.07417777511808607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,512,0.12823022736443415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,768,0.15656177202860513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,32,0.01737511157989502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,256,0.11327821678585476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,65536,64,0.02204888893498315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,32,0.09676622019873725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,64,0.09684444136089748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,65536,128,0.10672799746195476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,12288,0.477450688680013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,10240,0.602427535586887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,16384,0.6967164675394694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,12288,0.7466524442036947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,16384,0.9780320061577691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,10240,0.38299910227457684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,8192,0.4963048828972711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,8192,0.32141155666775173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,7168,0.38985511991712785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,6144,0.3476586606767442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,7168,0.2723795572916667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,6144,0.2209999958674113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,5120,0.199891553984748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,5120,0.31354043218824595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,3584,0.20572355058458117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,4096,0.1581911113527086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,4096,0.25724977917141384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,3072,0.17936355537838408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,3072,0.12123110559251572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,2560,0.09847288661532932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,2560,0.15310666296217176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,65536,2.236447016398112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,3584,0.13923554950290257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,1536,0.08146044280793932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,2048,0.08379289176728989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,2048,0.12095733483632405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,1024,0.056242664655049644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,768,0.043416887521743774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,512,0.033731556600994535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,768,0.04582311047448052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,512,0.04102399945259094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,1024,0.058950225512186684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,1536,0.07243644528918795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,128,0.008623999853928884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,256,0.01348355578051673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,64,0.007127111156781514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,32,0.0058622223635514574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,256,0.03668355610635545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,128,0.0329751107427809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,32,0.031649778286616005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,16384,64,0.032009777095582746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,16384,0.48997243245442706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,12288,0.5653937657674154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,12288,0.36940622329711914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,16384,0.6913866466946073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,10240,0.4675146738688151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,16384,65536,3.7069778442382812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,8192,0.3574835459391276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,8192,0.24475110901726616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,7168,0.3109182251824273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,10240,0.3046728769938151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,7168,0.21148622035980225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,6144,0.2741599877675374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,6144,0.1893306705686781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,5120,0.22377154562208387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,4096,0.1789004405339559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,3584,0.1538275612725152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,5120,0.1653546624713474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,3072,0.1294462283452352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,3584,0.10573866632249619
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,4096,0.12452356020609538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,3072,0.09509955512152778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,2560,0.10874399873945449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,2048,0.08811644713083903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,1536,0.06801155540678236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,2560,0.08814666668574016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,65536,1.8744142320421007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,768,0.03448266784350077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,1536,0.06137600209977892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,512,0.019433778193261888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,2048,0.07506666580835979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,768,0.04143822193145752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,256,0.012122666670216454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,512,0.036990222003724836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,1024,0.04880355464087593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,64,0.006711111300521427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,128,0.00816888858874639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,1024,0.045956446064843066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,256,0.0331582228342692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,32,0.005449777675999536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,128,0.030177778667873804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,32,0.028263999356163874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,12288,64,0.028594666057162817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,12288,65536,2.626329845852322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,12288,0.45560089747111004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,12288,0.36444711685180664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,10240,0.3581822183397081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,16384,0.5293582280476888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,16384,0.5949946509467231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,8192,0.29034577475653756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,7168,0.2512257893880208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,8192,0.23615556293063691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,6144,0.22299199634128145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,10240,0.3272293408711751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,7168,0.2139457729127672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,4096,0.16944444179534912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,5120,0.14568532837761775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,6144,0.18743022282918295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,5120,0.2015413310792711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,3072,0.1266097757551405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,3584,0.1589448849360148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,4096,0.12145422564612494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,2560,0.10718399948543972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,3072,0.09337066941791111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,3584,0.10930666658613418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,1536,0.06735377841525607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,2560,0.08540978034337361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,2048,0.09343110852771336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,2048,0.07107288969887628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,1024,0.04692622356944614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,768,0.02436888880199856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,512,0.019125334090656705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,65536,1.8617911868625219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,1536,0.060712887181176074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,256,0.011795555551846823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,1024,0.04412177867359585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,128,0.00794577764140235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,512,0.03598577777544657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,64,0.006280000011126201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,768,0.03995733459790548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,256,0.0319217774603102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,128,0.02906755606333415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,32,0.004903111192915174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,10240,65536,2.241210725572374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,32,0.027476444840431213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,10240,64,0.027671999401516382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,16384,0.4585750897725423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,12288,0.3652657667795817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,16384,0.3920835653940837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,12288,0.29702043533325195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,10240,0.31082844734191895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,10240,0.21257600519392225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,8192,0.16482933362325033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,7168,0.20089511076609293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,7168,0.15331466992696127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,8192,0.2741004361046685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,6144,0.17419466707441542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,6144,0.11487555503845215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,5120,0.1341751151614719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,4096,0.12025422520107693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,3584,0.09399200148052639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,5120,0.10031733247968887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,3072,0.0903644429312812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,65536,1.3502737681070964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,3072,0.06327911218007405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,2560,0.06852533419926961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,4096,0.0836026668548584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,3584,0.06999999947018094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,2048,0.05615555577807956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,1536,0.042487111356523305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,2560,0.0588888857099745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,1024,0.02238666680124071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,2048,0.04833777745564779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,512,0.011423110961914062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,768,0.01532444523440467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,1536,0.03415288858943515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,1024,0.02963288956218296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,256,0.00756444450881746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,128,0.005585777676767773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,65536,1.7944552103678386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,512,0.025787555509143408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,64,0.0046160001721647055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,768,0.027401778433057997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,8192,32,0.00433955548538102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,256,0.022679999470710754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,32,0.019837333096398246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,64,0.0203457772731781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,8192,128,0.020854221449957952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,12288,0.3198666572570801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,16384,0.3215431107415093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,16384,0.44093778398301864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,10240,0.24617687861124674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,12288,0.25870490074157715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,10240,0.1981431113349067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,8192,0.20264800389607748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,7168,0.17823378245035806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,8192,0.1373866664038764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,7168,0.11717866526709662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,5120,0.11657778422037761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,4096,0.09613422552744548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,6144,0.16471821731991237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,6144,0.09872977601157294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,5120,0.08409600125418769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,2560,0.06076444519890679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,3072,0.08082577917310926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,4096,0.07044266992145114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,3584,0.06422577963935004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,3072,0.05703910854127672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,1536,0.040114667680528425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,3584,0.09330044190088908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,2560,0.050917334026760526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,2048,0.05522844526502821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,768,0.014931556251313953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,2048,0.03925155599912008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,1536,0.03271466824743483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,65536,1.2446497811211479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,1024,0.01867911054028405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,512,0.010953777366214328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,256,0.007347555624114142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,128,0.005410666680998272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,1024,0.02823466724819607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,64,0.004454222404294544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,512,0.02456444501876831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,32,0.003956444561481476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,768,0.02642044425010681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,256,0.02145599987771776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,128,0.019465777609083388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,64,0.018664000762833487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,7168,65536,1.5980684492323134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,7168,32,0.019558222757445443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,12288,0.19911466704474556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,10240,0.21230044629838732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,16384,0.2766702175140381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,12288,0.2754106786515978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,16384,0.3685937722524007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,8192,0.17549333307478163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,7168,0.1456106636259291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,8192,0.11370933055877686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,7168,0.10003555483288235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,6144,0.12458666165669759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,5120,0.10160266690784031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,6144,0.08625600073072646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,10240,0.1634248892466227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,5120,0.0753964450624254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,4096,0.08535466591517131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,3584,0.07188977797826131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,2560,0.05269066823853386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,3072,0.07084977626800537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,3584,0.06100355254279243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,65536,1.1274195777045357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,2048,0.04344088832537333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,4096,0.06541155444251166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,2560,0.04400444361898634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,1536,0.03888533181614346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,1024,0.018064889642927382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,65536,1.3634844885932074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,1536,0.03148444493611654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,2048,0.03640444411171807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,1024,0.027090667022599116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,512,0.01089777797460556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,768,0.014924445086055331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,3072,0.054430223173565336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,768,0.024527110987239416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,128,0.005283555636803309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,256,0.00721866637468338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,64,0.004400889078776042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,512,0.022553778356975977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,6144,32,0.003731555408901639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,256,0.01995822290579478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,64,0.01791822248035007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,128,0.018726223044925265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,6144,32,0.018711111611790128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,16384,0.23615998691982695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,16384,0.30651378631591797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,12288,0.162188450495402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,12288,0.23855643802218965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,10240,0.1995697816212972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,8192,0.1518577734629313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,7168,0.12236621644761826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,10240,0.12032443947262234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,8192,0.09745066695743138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,7168,0.08541422420077854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,6144,0.10871822304195827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,5120,0.0944675538274977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,4096,0.07472444242901273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,5120,0.06649777624342176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,3584,0.06553244590759277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,4096,0.056369774871402316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,3072,0.05762488974465264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,3584,0.05091199941105313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,2048,0.038908445172839694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,2560,0.06532355811860827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,3072,0.04370133413208855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,1536,0.02862666712866889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,65536,1.0306711196899414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,1024,0.01794399983353085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,6144,0.08232800165812175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,2048,0.03566311134232415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,65536,1.1319875717163086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,512,0.01071111112833023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,768,0.01460444430510203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,256,0.007108444141017065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,128,0.005218666460778978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,1536,0.031096001466115315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,1024,0.02641955514748891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,768,0.024131556351979572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,512,0.022429333792792425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,256,0.01998222205373976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,32,0.0032968889507982465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,5120,64,0.004306666553020477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,32,0.017599110802014668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,2560,0.039396444956461586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,64,0.018008889423476327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,5120,128,0.018817777434984844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,16384,0.25052444140116376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,12288,0.19187466303507486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,16384,0.1846524477005005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,12288,0.12494577301873101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,8192,0.11232888698577881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,10240,0.16540800200568304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,8192,0.07727733585569593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,10240,0.10089600086212158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,7168,0.09762577878104316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,6144,0.08539733621809219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,4096,0.05942844682269626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,6144,0.06109778086344401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,7168,0.07330577903323703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,3584,0.05284533235761854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,5120,0.07975111405054729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,4096,0.04390933447413974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,65536,0.9256132973564996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,3072,0.04636711213323805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,3584,0.03917866614129808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,65536,0.8983910878499349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,2560,0.03469244307941861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,2048,0.021592888567182753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,1536,0.017504889104101393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,5120,0.05724710888332791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,3072,0.0339680016040802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,2048,0.02718933257791731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,1024,0.01239377756913503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,1024,0.020502222908867728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,2560,0.03085955646302965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,512,0.00797155582242542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,768,0.010237333675225576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,256,0.005323555734422471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,128,0.00425155543618732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,768,0.019104889697498746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,512,0.01742577718363868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,64,0.0035297779573334586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,1536,0.024331554770469666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,4096,32,0.0030053332448005676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,128,0.014463999205165438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,64,0.014070222775141398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,256,0.016032889485359192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,4096,32,0.014053333136770459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,16384,0.2141351170010037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,10240,0.13055022557576498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,12288,0.1584871080186632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,16384,0.16439199447631836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,8192,0.10875378052393596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,12288,0.10449955860773723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,10240,0.08745600117577447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,8192,0.07172977924346924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,7168,0.09832888841629028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,6144,0.087826669216156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,7168,0.0638515551884969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,5120,0.06377777788374159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,4096,0.05272355675697327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,6144,0.05708622270160251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,5120,0.04990933338801066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,3584,0.046626667181650795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,4096,0.038324445486068726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,65536,0.8133271005418566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,65536,0.8343831168280708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,2048,0.01995377739270528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,2560,0.03009777930047777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,3072,0.03589422172970242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,2560,0.02918222215440538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,1536,0.01589866644806332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,1024,0.01145600030819575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,3584,0.03499555587768555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,3072,0.03201866812176175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,512,0.007245333658324347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,2048,0.025797333982255723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,1536,0.02307111190425025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,256,0.005347555710209741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,1024,0.02000177734427982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,128,0.0041306668685542215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,768,0.018383110562960308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,768,0.009578666753239101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,64,0.0034728890491856467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,512,0.016696888539526198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3584,32,0.00332266671790017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,128,0.014027555783589682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,64,0.013658666776286231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,32,0.013562666873137156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3584,256,0.015136000182893542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,12288,0.13379377788967556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,16384,0.19736977418263754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,16384,0.13830133279164633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,10240,0.1181466711892022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,10240,0.07951111263699003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,12288,0.09497155745824178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,8192,0.08766044510735406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,7168,0.07885689205593534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,8192,0.06679555442598131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,6144,0.0684408876630995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,5120,0.05780533287260267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,7168,0.058989332781897656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,4096,0.04654399885071648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,6144,0.0527724458111657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,5120,0.04520000020662943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,65536,0.7172515657213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,4096,0.03472000029351976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,2560,0.022036444809701707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,2048,0.017775111728244357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,3584,0.04055288765165541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,3584,0.03290133343802558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,3072,0.03352177805370755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,1536,0.014543111125628153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,2560,0.02735822233888838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,3072,0.030061334371566772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,1024,0.010379555324713389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,2048,0.024547555380397375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,65536,0.7891982396443685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,512,0.0068417779273457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,1536,0.021687999367713928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,768,0.008716444174448649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,1024,0.01885333326127794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,768,0.017403556240929496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,512,0.016178665889634024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,128,0.003939555750952827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,64,0.003206222214632564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,32,0.003093333293994268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,256,0.014639111028777229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,3072,256,0.005111111121045219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,128,0.013400000002649097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,64,0.01308444473478529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,3072,32,0.012861332959598966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,12288,0.12315911716885036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,16384,0.16042666965060765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,10240,0.08887377712461685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,12288,0.08618133597903782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,8192,0.07261155711279975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,16384,0.11672266324361165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,10240,0.0742924412091573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,7168,0.0659360024664137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,6144,0.05531111028459337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,8192,0.061016890737745494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,65536,0.594347529941135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,6144,0.04847200049294365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,5120,0.04893777767817179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,7168,0.05418133404519823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,3584,0.0358168880144755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,4096,0.0402782228257921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,3072,0.02147644427087572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,5120,0.039183109998703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,2560,0.01918577816751268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,65536,0.7511306868659126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,3584,0.031621333625581526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,2048,0.0170817772547404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,3072,0.029102222787009344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,4096,0.03416266706254747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,1536,0.013678222066826291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,1024,0.010144888526863521
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,2560,0.02645422187116411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,2048,0.02377333409256405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,512,0.006670222100284364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,768,0.008482666479216682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,1536,0.02123466630776723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,256,0.004941333499219683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,1024,0.018453333112928603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,128,0.0036640001667870414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,768,0.017024000485738117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,64,0.003015999992688497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2560,32,0.0028604444944196274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,512,0.015608888533380298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,128,0.013150222599506378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,256,0.014196443888876172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,64,0.012816888590653738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2560,32,0.012719111310111152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,16384,0.10894133647282918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,12288,0.08375200298097397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,10240,0.070905778143141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,12288,0.07862222194671631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,16384,0.10883822043736775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,8192,0.058394663863711886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,10240,0.0679137772983975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,7168,0.05144266618622673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,8192,0.056812445322672524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,5120,0.038350221183564924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,65536,0.5039457745022243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,6144,0.045026666588253446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,7168,0.049024889866511025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,4096,0.022916444473796423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,5120,0.038253333833482527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,3584,0.019327110714382596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,3072,0.017041777571042378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,6144,0.043423109584384494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,4096,0.033117334047953285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,2560,0.01789866718981001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,3584,0.030642665094799463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,2048,0.01573511130279965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,1536,0.011944889194435544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,3072,0.028248889578713313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,1024,0.008929777476522658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,65536,0.7276150915357801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,2560,0.025467554728190105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,768,0.007511999871995714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,1536,0.0203128887547387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,512,0.006004444426960415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,256,0.004543111142185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,128,0.00360000009338061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,768,0.016065778003798593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,64,0.0028942223224374983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,1024,0.01759111053413815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,512,0.014735110931926303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,2048,0.022946667340066697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,256,0.013352000051074557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,2048,32,0.0026648888985315957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,128,0.012285333540704517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,64,0.011873777541849347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,2048,32,0.011839110818174152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,12288,0.0648684435420566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,16384,0.08330044481489393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,10240,0.054915553993648954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,16384,0.09848266839981079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,8192,0.04545422063933479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,12288,0.07454578081766765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,7168,0.04208711120817396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,10240,0.0639573335647583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,65536,0.37873421774970156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,8192,0.05303200085957845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,6144,0.03629866573545668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,4096,0.017990221579869587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,5120,0.021643555826610986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,6144,0.04271022147602505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,3584,0.017306667235162523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,5120,0.037520888778898455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,4096,0.03240711159176297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,7168,0.04781422350141737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,3072,0.017264000243610807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,3584,0.0299653344684177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,2048,0.013636444177892474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,3072,0.02751733362674713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,65536,0.697337786356608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,1536,0.010312000082598792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,2560,0.015483554866578845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,1024,0.007756444315115611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,2560,0.024858666790856257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,768,0.006548444430033366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,2048,0.02214755614598592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,1536,0.019658666517999437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,512,0.005392888767851724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,256,0.0040942223535643685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,128,0.0032906666811969546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,64,0.002623999905255106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,1024,0.016731555263201397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,768,0.015392889579137167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,256,0.012599111431174807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,512,0.014070222775141398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1536,32,0.0024888888001441956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,128,0.01165066659450531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,64,0.011171555353535546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,16384,0.05759021970960829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,12288,0.04469688733418783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1536,32,0.011163555913501315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,8192,0.027448000179396734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,10240,0.0381368895371755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,16384,0.09390755494435628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,7168,0.01904977858066559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,65536,0.27237778239780003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,12288,0.07336977455351087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,10240,0.06303377946217854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,6144,0.019508444600635104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,8192,0.05239466826121012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,4096,0.014304000470373364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,7168,0.04766844378577339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,5120,0.01661422186427646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,3584,0.013447999954223633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,3072,0.012292444705963135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,5120,0.037259555525249906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,2560,0.011056889262464313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,4096,0.03189511100451151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,6144,0.042575998438729175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,3584,0.029711110724343195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,2048,0.009896888501114314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,3072,0.026923555466863845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,1536,0.008239111138714684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,1024,0.006414222220579783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,2560,0.024375110864639282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,768,0.005547555370463266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,2048,0.02161688937081231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,65536,0.6544115808275011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,512,0.004586666822433472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,1536,0.01939911146958669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,1024,0.01629333363638984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,256,0.003677333394686381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,128,0.003059555465976397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,768,0.01517777807182736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,64,0.0025680000997251934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,1024,32,0.00231200001306004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,512,0.013766222529941134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,128,0.011217777927716574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,64,0.010813333094120026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,32,0.010871999793582492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,16384,0.0450835559103224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,12288,0.034210665358437434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,10240,0.0196088883611891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,65536,0.21969332959916857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,16384,0.09418844514422947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,1024,256,0.012280888855457306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,8192,0.016484444340070088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,12288,0.07337866889105903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,10240,0.0628239976035224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,7168,0.018985778093338013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,6144,0.017050666941536795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,5120,0.014684443672498068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,8192,0.05246666736072964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,7168,0.04747466577423943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,6144,0.0425217780801985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,4096,0.012576888832781049
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,3584,0.012110222544935016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,5120,0.03724977705213759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,4096,0.03190844588809543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,3072,0.01109866632355584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,2560,0.009944889280531142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,2048,0.009850666754775578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,65536,0.6417102283901638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,3584,0.029586666160159644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,3072,0.026946667167875502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,1536,0.008228444390826756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,2560,0.02424533334043291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,1024,0.0063688887490166565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,2048,0.021852445271280076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,512,0.004584889031118817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,768,0.005585777676767773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,1536,0.019190222024917603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,1024,0.01625333395269182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,256,0.003648888733651903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,768,0.01481066644191742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,128,0.002979555477698644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,64,0.0023235556566052968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,512,0.013691555294725629
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,768,32,0.002193777718477779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,256,0.01221866657336553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,128,0.011180444724029966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,64,0.010808000134097205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,768,32,0.01090755561987559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,12288,0.01908622185389201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,10240,0.016894222961531747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,16384,0.03186400069130792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,65536,0.15287200609842935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,8192,0.01457244488928053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,16384,0.09387377897898357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,12288,0.07315999931759305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,7168,0.01312622262371911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,6144,0.011997333003415002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,10240,0.06301422251595391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,5120,0.011782222323947482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,8192,0.052609778112835355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,4096,0.010092444717884064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,7168,0.04760355419582791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,6144,0.04231377773814731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,5120,0.03716622127426995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,3072,0.008664888640244802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,3584,0.009460444251696268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,2560,0.007921777665615082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,3584,0.029480000336964924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,2048,0.006978666616810693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,3072,0.02684177789423201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,1536,0.00924711094962226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,2560,0.024497777223587036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,4096,0.03208177619510227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,1024,0.006161777923504512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,2048,0.021697777840826247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,65536,0.6532417933146158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,768,0.005405333307054307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,512,0.004401777767472797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,1536,0.019158222609096102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,1024,0.016233777006467182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,128,0.002593777866827117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,256,0.0032746667663256326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,64,0.002135111019015312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,768,0.014956444501876831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,512,32,0.0023795554621352092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,256,0.012287110918098025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,512,0.013601777454217276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,128,0.011141333315107556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,64,0.01075466639465756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,16384,0.016556445095274184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,65536,0.05530666642718845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,512,32,0.010795555180973478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,12288,0.013636444177892474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,10240,0.01204711116022534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,8192,0.01016533292002148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,7168,0.00959022260374493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,16384,0.093685335583157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,6144,0.0088755554623074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,12288,0.07316444317499797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,5120,0.007983999947706858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,8192,0.052490668164359204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,7168,0.04740000102255079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,10240,0.06299200322892931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,4096,0.007035555938879649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,3584,0.00685777763525645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,6144,0.04240533378389147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,3072,0.0063635553750726916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,5120,0.03699466586112976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,2560,0.006039999839332368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,4096,0.031876444816589355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,2048,0.005425777700212266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,3584,0.029465778006447688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,3072,0.026847999956872728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,1536,0.008879111044936711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,1024,0.006487111250559489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,2560,0.02439466615517934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,2048,0.021519111262427434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,768,0.005347555710209741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,512,0.0041680000722408295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,1536,0.018986667195955913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,65536,0.6570106612311469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,1024,0.016050666570663452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,256,0.003010666618744532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,768,0.0148062225845125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,64,0.0019706665641731685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,128,0.0023884444187084832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,512,0.013452444639470843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,256,32,0.002181333386235767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,256,0.011957333319716983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,128,0.011136000355084738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,65536,0.02975289026896159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,16384,0.010464000205198923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,64,0.010806221928861407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,12288,0.008904000123341879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,10240,0.008028444316652086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,256,32,0.010864888628323873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,8192,0.007244444555706448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,7168,0.006940444310506185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,6144,0.0064702220261096954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,16384,0.09361777702967326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,12288,0.07334044244554308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,10240,0.062845336066352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,8192,0.05256888932651944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,5120,0.005958222266700532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,4096,0.005491555564933353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,7168,0.04749422272046407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,3584,0.005619555711746216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,6144,0.04255377915170458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,5120,0.03718666566742791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,3072,0.005134222408135732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,4096,0.03190844588809543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,2560,0.0064275554484791225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,2048,0.005694222119119432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,3584,0.029343111647499934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,1536,0.0047564444442590075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,3072,0.026830222871568467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,1024,0.0057591112951437635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,768,0.004855999930037392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,2560,0.024363555841975745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,2048,0.021568889419237774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,512,0.0037928890022966596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,1536,0.019010666343900893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,65536,0.6499244372049967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,256,0.0028648889727062653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,1024,0.016183111402723525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,128,0.0023813332534498638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,768,0.014719110396173267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,64,0.0021466666625605687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,128,32,0.0021217777911159727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,512,0.013495999905798169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,256,0.011971555650234222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,65536,0.030368000268936157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,16384,0.010999999940395355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,64,0.010966222319338055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,128,0.011108444796668159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,12288,0.009232000344329411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,128,32,0.010696888797812991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,10240,0.008552889029184977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,8192,0.0074728892909155945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,7168,0.007179555793603261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,16384,0.0933519999186198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,6144,0.006723555425802867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,12288,0.07309333483378093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,10240,0.06259110901090834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,8192,0.052226665947172374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,7168,0.04733599887953865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,5120,0.0063795554969045855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,4096,0.005693333430422678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,3584,0.005622222191757626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,6144,0.04199111130502489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,3072,0.005151999907361137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,5120,0.03690133492151896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,4096,0.03160888950030009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,2560,0.004971555537647671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,3584,0.029288000530666772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,2048,0.004504000147183736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,1536,0.004101333518822988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,3072,0.026657778355810378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,1024,0.005571555760171678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,2560,0.02405244443151686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,768,0.004616888860861461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,512,0.003680888977315691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,2048,0.02128622266981337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,1536,0.01889955500761668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,1024,0.015811555915408664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,256,0.002756444530354606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,65536,0.6704924371507434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,128,0.0022533333135975734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,64,0.0021582223061058256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,768,0.014595554934607612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,64,32,0.002128888956374592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,512,0.013296888934241401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,256,0.01183555523554484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,65536,0.01774311065673828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,128,0.010935111178292168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,16384,0.007839110990365347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,12288,0.007000000112586551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,64,0.010831111007266574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,64,32,0.010639111200968424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,10240,0.006631111105283101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,8192,0.006057777752478917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,16384,0.09322577714920044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,7168,0.005734222216738596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,12288,0.07200977537367079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,6144,0.0052168890833854675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,10240,0.06218666500515408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,8192,0.05237244566281637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,5120,0.005103999955786599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,4096,0.004978666702906291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,7168,0.04693333307902018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,6144,0.041668444871902466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,3584,0.004865777575307422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,5120,0.03664177656173706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,3072,0.004456000195609199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,2560,0.004654222064548069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,4096,0.031375110149383545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,2048,0.003936000168323517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,1536,0.0038000001675552795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,3584,0.02895555562443203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,3072,0.026591110560629103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,1024,0.003528888854715559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,2560,0.023913777536816064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,2048,0.02138311167558034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,768,0.004554666578769684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,65536,0.6538773112826878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,1536,0.01867644488811493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,512,0.003590222034189436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,256,0.002687999978661537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,128,0.002206222257680363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,64,0.0021404443929592767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,1024,0.015825778245925903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,768,0.014430221584108142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,512,0.013208000196350945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,256,32,32,0.0021102221475707158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,128,0.011034666664070554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,256,0.0118568887313207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,64,0.010641777680979835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,256,32,32,0.010640888578361936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,10240,1.4167733722262914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,12288,1.696298705206977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,8192,1.8140782250298395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,16384,2.192928949991862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,10240,2.214920891655816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,8192,1.0903529061211479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,7168,0.9303493499755859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,12288,2.8983902401394315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,6144,0.8148479991488986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,6144,1.391996489630805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,5120,0.6365804672241211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,7168,1.5694079928927953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,16384,3.504615147908529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,5120,1.2431395848592122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,4096,0.5522969033983018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,4096,0.9236506356133355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,3072,0.40507645077175564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,2560,0.5766337712605795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,2560,0.35225510597229004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,2048,0.2945840093824598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,3072,0.6864764425489637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,3584,0.49402666091918945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,3584,0.7964942190382215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,2048,0.5019831127590603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,512,0.11974844667646621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,1024,0.2609822220272488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,256,0.06640622350904676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,768,0.1501679950290256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,1536,0.24731289015875924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,1536,0.3892880015903049
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,768,0.21691021654340956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,1024,0.18703199757470024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,512,0.1227031151453654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,32,0.022557333111763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,64,0.020020445187886555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,65536,128,0.03885422150293986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,256,0.10914844274520874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,32,0.09460977713267009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,128,0.10358222325642903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,65536,64,0.09551288684209187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,10240,0.4084995587666829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,12288,0.5145209100511339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,10240,0.5781084696451823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,16384,0.625632021162245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,12288,0.7516115506490072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,16384,0.9638986587524414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,8192,0.29529865582784015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,7168,0.25692799356248647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,7168,0.382992876900567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,6144,0.34129778544108075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,8192,0.47820530997382266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,5120,0.28103378083970815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,6144,0.22762934366861978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,4096,0.2513217661115858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,3072,0.19029511345757377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,4096,0.15381599797142878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,5120,0.19288799497816297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,3584,0.2180960045920478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,3584,0.1359422206878662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,3072,0.11733688248528375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,65536,2.063870324028863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,2560,0.15175644556681314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,2048,0.08293066422144572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,2048,0.12047199408213298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,2560,0.10352710882822673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,1536,0.09188889132605658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,512,0.02649333410792881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,768,0.04331199990378486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,1024,0.06211733155780368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,256,0.011217777927716574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,768,0.04467288984192742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,1024,0.055128888951407544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,512,0.039905776580174766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,1536,0.07245066430833605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,128,0.03243111239539252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,64,0.006743999818960826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,128,0.00851466672288047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,256,0.03638222151332431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,64,0.03110577662785848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,32,0.005425777700212266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,16384,32,0.03109333250257704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,12288,0.360210657119751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,16384,0.46933868196275497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,12288,0.6030969089931911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,16384,0.7036693361070422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,16384,65536,3.439242680867513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,10240,0.2948835425906711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,10240,0.4175146685706244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,8192,0.33731378449334043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,8192,0.24386310577392578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,7168,0.21129155158996582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,7168,0.3080657853020562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,6144,0.2705457740359836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,6144,0.1715102195739746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,4096,0.176801774236891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,5120,0.14037599828508165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,4096,0.11470399962531196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,5120,0.23518843120998809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,3584,0.1525297827190823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,3584,0.10400533013873631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,3072,0.12874310546451145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,65536,1.7062613169352214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,2560,0.10849689112769233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,2560,0.08159999714957343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,1536,0.0677200025982327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,1024,0.05084888802634346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,2048,0.07075466712315877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,1536,0.06014044417275322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,2048,0.09476621945699056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,512,0.017533333765135873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,1024,0.04424266682730781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,256,0.011688888900809817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,512,0.03631822268168131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,128,0.007994666695594788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,768,0.04000088903639052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,64,0.006693333387374878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,3072,0.0976782242457072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,768,0.03269244564904107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,256,0.03219288918707106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,32,0.004575110971927643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,128,0.02980888883272807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,64,0.02822222312291463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,12288,32,0.02827022141880459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,12288,65536,2.624891493055556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,12288,0.35552533467610675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,16384,0.5594862302144369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,12288,0.44884623421563047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,16384,0.4623102082146539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,10240,0.2869573434193929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,10240,0.35191644562615293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,8192,0.2869475682576497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,8192,0.23164444499545625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,6144,0.262389341990153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,7168,0.300197336408827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,7168,0.20135645071665445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,6144,0.16748266749911836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,5120,0.21530666616227892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,4096,0.11250489287906223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,3072,0.09810044368108113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,5120,0.14528621567620173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,3584,0.12105600039164226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,3584,0.10193244616190593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,2560,0.08199555344051786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,4096,0.1820248895221286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,2048,0.06667911344104342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,1536,0.05137688914934794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,2560,0.08011200030644734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,1024,0.03619111246532864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,2048,0.06905688842137654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,3072,0.09475821918911403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,768,0.018372444642914664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,512,0.013532444834709167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,1024,0.04284800092379252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,1536,0.059947556919521756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,256,0.009204444785912832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,512,0.03547911180390252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,64,0.005676444619894028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,128,0.006824888702895906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,256,0.031116445859273274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,65536,1.6841004689534504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,32,0.0041333333485656315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,128,0.028749333487616643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,64,0.027289777994155884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,32,0.026625777284304302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,10240,768,0.03884889019860162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,10240,65536,2.1886204613579645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,12288,0.3579422103034125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,12288,0.25097865528530544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,10240,0.27833689583672416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,16384,0.3292142285241021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,16384,0.4924489127265082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,10240,0.20480267206827799
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,8192,0.2283102141486274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,7168,0.19703111383650038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,6144,0.1708364486694336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,8192,0.1700906621085273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,5120,0.1341760026084052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,7168,0.1417484415902032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,5120,0.08932444122102524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,65536,1.1798346837361653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,4096,0.12109688917795818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,6144,0.10669422149658203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,3584,0.06770844591988458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,3584,0.10442844364378188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,2560,0.06857422325346205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,3072,0.06615733438067965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,4096,0.08148800002204047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,2560,0.054099556472566396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,65536,1.7394293679131403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,1024,0.018076444665590923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,2048,0.04626489016744825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,1536,0.03312888741493225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,1536,0.04723555511898465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,768,0.015063110325071545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,3072,0.09026844633950128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,512,0.01094133324093289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,2048,0.062013334698147245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,128,0.005336888962321811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,256,0.007408888803588019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,512,0.02528088953759935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,256,0.022602667411168415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,64,0.004464888738261329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,8192,32,0.0037751110891501107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,1024,0.02955199943648444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,64,0.01983022193113963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,128,0.020408888657887776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,768,0.026733333865801494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,8192,32,0.019923556182119582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,12288,0.220923556221856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,12288,0.31464799245198566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,16384,0.29528355598449707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,10240,0.2441075642903646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,10240,0.17834756109449598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,16384,0.43360267745123965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,8192,0.20063377751244438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,7168,0.17303110493554008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,7168,0.11045155260297988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,8192,0.13209511174096003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,6144,0.14828177293141684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,5120,0.11672177579667832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,4096,0.09690222475263809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,4096,0.06806311342451307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,6144,0.10215022166570027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,3584,0.08281777964697944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,3072,0.05558844407399496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,5120,0.0879582232899136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,2048,0.049472000863817006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,65536,1.07477781507704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,2560,0.061103112167782254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,3072,0.0740471084912618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,2560,0.050604446066750415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,1536,0.04107466671201918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,3584,0.06638044118881226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,2048,0.03760800096723769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,1024,0.018111111389266122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,768,0.014946666028764514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,512,0.010769777827792697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,256,0.007231111327807109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,128,0.005236444373925527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,1024,0.027460444304678176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,1536,0.03186755710177951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,768,0.025598223010698955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,32,0.00332266671790017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,64,0.004565333326657613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,512,0.023579556081030104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,7168,65536,1.5542462666829426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,256,0.02164266672399309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,64,0.01870666609870063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,32,0.01887911061445872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,7168,128,0.02012622190846337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,10240,0.20974311563703749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,12288,0.189665781127082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,12288,0.2713128990597195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,10240,0.14643732706705728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,16384,0.2570088969336616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,16384,0.34168267250061035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,8192,0.17358133527967665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,7168,0.14286133978101942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,8192,0.11103110843234593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,7168,0.09595466984642877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,6144,0.08424711227416992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,5120,0.10109155707889134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,4096,0.08410844537946914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,3584,0.07181421915690105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,6144,0.13989688290490046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,3072,0.06339999702241686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,4096,0.0654088921017117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,2560,0.05298133360015022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,5120,0.07940000295639038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,65536,0.9703786638047961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,3584,0.05589422252443102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,3072,0.05357955561743843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,2048,0.04399289025200737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,2560,0.040864888164732195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,1536,0.03584533267550998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,2048,0.03519644339879354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,768,0.014825777875052558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,65536,1.3205146789550781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,1024,0.017955554856194388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,1536,0.030600888861550227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,1024,0.026399999856948853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,512,0.01068800025516086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,256,0.007060444189442529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,128,0.005195555587609609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,64,0.0040737779604064096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,6144,32,0.003079111170437601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,128,0.01843644513024224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,64,0.01808355583084954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,512,0.0222382214334276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,32,0.018613333503405254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,768,0.024327110913064744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,6144,256,0.020086222224765353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,16384,0.28307199478149414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,12288,0.2293626732296414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,12288,0.15169066852993435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,16384,0.22613067097134062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,10240,0.2090266678068373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,8192,0.14532000488705105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,7168,0.11263022157880996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,10240,0.11830222606658936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,8192,0.094449778397878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,7168,0.08901421891318427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,5120,0.08103733592563205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,65536,0.8613271183437772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,6144,0.11195733812120225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,4096,0.07358933157391019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,3584,0.05770844221115112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,5120,0.06399555338753594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,65536,1.0997716055976021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,4096,0.05477689041031731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,3584,0.049512889650132924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,2560,0.04208533300293816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,2048,0.034373333056767784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,3072,0.04218222366438972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,2560,0.03859733210669623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,1536,0.016720000240537856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,3072,0.056207113795810275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,768,0.009659555223253038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,1024,0.01181688904762268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,2048,0.03457244568400913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,6144,0.07883555359310575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,768,0.023792889383104112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,1536,0.03021777669588725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,512,0.007576888634098901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,256,0.005482666608360078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,512,0.02162133322821723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,128,0.0041626666982968645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,1024,0.02593066626124912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,256,0.019461333751678467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,64,0.003585777762863371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,5120,32,0.002921777880854077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,64,0.01826844447188907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,128,0.017988445030318368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,5120,32,0.01737600068251292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,16384,0.15659733613332114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,16384,0.2412355475955539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,10240,0.08801777495278253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,10240,0.14408444033728704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,12288,0.1059066653251648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,12288,0.1870888868967692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,8192,0.1111804379357232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,7168,0.0983733336130778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,6144,0.08528177605734931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,5120,0.0718382199605306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,7168,0.06423733631769817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,8192,0.07238755623499553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,6144,0.05688533518049452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,5120,0.050373332368003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,4096,0.05969599882761637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,3584,0.052240888277689614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,4096,0.04111466805140177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,3072,0.04036800066630045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,2560,0.03381244341532389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,2048,0.01937866707642873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,3584,0.03171822097566392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,1536,0.014427555931939019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,3072,0.029040889607535467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,2560,0.02609599961174859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,1024,0.010415111151006486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,2048,0.023702222439977858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,512,0.007127111156781514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,1536,0.021292444732454088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,768,0.008608000146018134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,256,0.005050666630268097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,65536,0.8971190982394748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,1024,0.01849600010448032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,65536,0.7930817604064941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,128,0.0038648889296584656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,32,0.0030622221529483795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,256,0.014363555444611443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,4096,64,0.0032257777121331957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,128,0.013100444442696042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,768,0.0170817772547404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,32,0.013036444783210754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,512,0.0159262220064799
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,4096,64,0.013202666408485837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,12288,0.1565742227766249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,16384,0.1346044407950507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,12288,0.09650222460428874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,16384,0.23390489154391816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,10240,0.13548355632358128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,10240,0.08127288685904609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,7168,0.09691644377178615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,6144,0.07639466391669379
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,8192,0.09932621982362534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,8192,0.06765244404474895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,65536,0.6719697846306695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,5120,0.0651057759920756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,65536,0.7946800125969781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,4096,0.052575111389160156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,7168,0.06400266620847914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,6144,0.05311288767390781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,5120,0.045808000697029956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,3584,0.046431110964881055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,3072,0.035263111193974815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,2560,0.02728355593151516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,4096,0.032940444019105695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,1536,0.013721778160995908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,1024,0.010103111465771994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,2560,0.025495999389224585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,2048,0.017072889539930556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,1536,0.02071466710832384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,768,0.008529778156015608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,3072,0.028071999549865723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,3584,0.03050222330623203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,512,0.006689777804745569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,1024,0.01791911158296797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,2048,0.02315555512905121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,128,0.003779555360476176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,256,0.005040888984998067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,64,0.003084444544381566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,768,0.016916443904240925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,512,0.015444444285498725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3584,32,0.0029288888391521242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,64,0.012891555825869242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,256,0.01423911088042789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,32,0.012744888663291931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3584,128,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,16384,0.11668177445729573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,12288,0.13180622789594862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,12288,0.08833599752850002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,16384,0.20832178327772352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,8192,0.08639111121495564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,10240,0.10554133521185981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,10240,0.07500800159242418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,8192,0.06283999813927545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,7168,0.08214310805002849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,7168,0.05565155877007378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,6144,0.06830044587453206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,65536,0.6935226652357313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,65536,0.6179084248012966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,5120,0.05728977918624878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,6144,0.04961599906285604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,4096,0.04751999841796028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,5120,0.04087377919091119
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,3584,0.040256887674331665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,3072,0.029816001653671265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,2560,0.020431111256281536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,3072,0.02777333392037286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,2048,0.01687022215790219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,1024,0.010029333333174387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,4096,0.03252799974547492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,768,0.008330666356616551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,2560,0.025225778420766194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,1536,0.020558221472634208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,3584,0.029987557066811457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,1536,0.013357333011097379
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,512,0.006586666736337874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,2048,0.023092443744341534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,128,0.0038453332251972626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,1024,0.01774133410718706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,256,0.004870222260554631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,768,0.016493333710564505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,512,0.01517955627706316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,32,0.002789333255754577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,3072,64,0.003006222140457895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,256,0.01366311146153344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,128,0.01293244461218516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,64,0.012544000314341651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,3072,32,0.012666666673289405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,12288,0.10530400276184082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,12288,0.07984533574846056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,16384,0.14716445075141057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,10240,0.08672000302208795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,16384,0.10613244771957397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,8192,0.07164000140296088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,7168,0.06440622276730008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,8192,0.0578133331404792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,6144,0.05489777856402927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,10240,0.06951555278566149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,7168,0.05061422122849358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,6144,0.043324444029066295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,4096,0.04015377826160855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,5120,0.04827910992834303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,65536,0.5887253549363878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,5120,0.035988443427615695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,65536,0.590761767493354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,3584,0.03427733315361871
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,3072,0.019553777244355943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,2560,0.017450667089886136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,1536,0.01313866674900055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,3584,0.028984887732399836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,4096,0.0313351121213701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,2048,0.015230221880806817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,1024,0.008789333204428354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,1536,0.019356444478034973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,768,0.007351111206743453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,3072,0.026718222432666357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,2048,0.021713778376579285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,2560,0.02424266603257921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,512,0.0058542220956749385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,1024,0.016699555847379897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,768,0.015558222929636637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,128,0.003588444242874781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,64,0.002830222249031067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,256,0.004463110946946674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,512,0.014381332529915703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2560,32,0.0026213334252436957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,128,0.011960888902346293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,256,0.012912888493802814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,64,0.011606222225560082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2560,32,0.011784888803958893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,12288,0.08316177792019315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,10240,0.0705946683883667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,16384,0.09682044718000625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,16384,0.12329155868954128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,12288,0.07289333475960626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,8192,0.05819466378953722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,7168,0.05703644620047676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,8192,0.05341777867741055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,10240,0.06368266873889498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,6144,0.04531288809246487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,7168,0.045055111249287925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,65536,0.48506487740410703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,5120,0.0392871101697286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,4096,0.023760888311598036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,6144,0.04013599952061971
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,3584,0.018696889281272888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,65536,0.5593253241644965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,5120,0.03553422292073568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,3584,0.028547555208206177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,2048,0.013691555294725629
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,4096,0.030971556901931763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,3072,0.026146666871176824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,3072,0.017095999585257638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,1024,0.007747555772463481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,1536,0.009983110758993361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,2560,0.023960888385772705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,2560,0.015450666348139444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,2048,0.021306667062971327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,1536,0.01891911029815674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,768,0.006536888993448681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,512,0.005410666680998272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,256,0.004685333205593957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,128,0.003343111111058129
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,64,0.002578666640652551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,768,0.01533422205183241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,1024,0.01621955633163452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,256,0.012536889149083031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,2048,32,0.002424000038041009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,128,0.011408000356621213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,512,0.013771555489963956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,64,0.01129066695769628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,2048,32,0.011351111034552256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,10240,0.054733332660463124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,16384,0.08299644125832452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,12288,0.06433066394593981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,8192,0.04525688952869839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,16384,0.08921866946750218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,12288,0.06884177525838216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,7168,0.037794666157828435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,10240,0.05960533354017469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,6144,0.03167288833194309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,8192,0.04965600040223864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,7168,0.04456177684995863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,4096,0.01614488826857673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,5120,0.018992889258596633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,65536,0.37264445092942977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,3584,0.015028445257080926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,3072,0.01351111133893331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,6144,0.03997244437535604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,5120,0.03535644544495477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,2560,0.012264000044928657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,4096,0.030623111459943984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,3584,0.028135110934575398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,2048,0.010606221854686737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,3072,0.025919111238585577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,65536,0.527932431962755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,2560,0.023579556081030104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,1024,0.006776000062624614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,2048,0.02092355489730835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,768,0.0058444444504049085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,1536,0.008737777670224508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,1536,0.01849244369400872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,512,0.004966222163703707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,1024,0.015847999188635085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,128,0.003047111133734385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,768,0.014863111906581454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,512,0.013356444736321768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,256,0.012255111502276527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,32,0.0023271110322740343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,128,0.011239111423492432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,64,0.0024862223201327855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1536,256,0.003973333372010125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,64,0.010832889212502373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1536,32,0.010768888725174798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,12288,0.044682668315039746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,16384,0.057364443937937416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,10240,0.035981333918041654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,8192,0.022660444180170696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,12288,0.06820533010694715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,16384,0.08756266699896918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,65536,0.26114755206637913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,7168,0.017500445246696472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,8192,0.04941777719391716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,10240,0.058796445528666176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,6144,0.016191111670600045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,5120,0.013752889302041797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,7168,0.04454400142033895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,4096,0.011894221935007306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,5120,0.035086221165127225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,65536,0.480055120256212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,3072,0.010514666636784872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,3584,0.027990221977233887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,6144,0.03983555568589105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,4096,0.030505776405334473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,2048,0.00978311151266098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,3072,0.025659556190172832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,2560,0.009369778136412302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,1536,0.00816533300611708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,1024,0.006342222293217977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,768,0.005535999933878581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,3584,0.011334222224023608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,2560,0.023352000448438857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,2048,0.02091288897726271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,1536,0.018358222312397428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,512,0.004579555657174853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,256,0.0037413334680928122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,128,0.002842666581273079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,768,0.014535110857751636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,64,0.0022533333135975734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,512,0.013439999686347114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,256,0.012115555504957834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,1024,32,0.002143111079931259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,1024,0.015855999456511605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,128,0.011085333095656501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,64,0.010751999914646149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,1024,32,0.010750222537252637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,16384,0.042506668302747935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,12288,0.03314755691422357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,10240,0.018211555149820115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,16384,0.08737778001361424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,8192,0.015252444479200574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,65536,0.20588978131612143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,12288,0.06824533144632976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,7168,0.015006222658687167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,6144,0.014008000493049622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,10240,0.05889155467351278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,5120,0.012257777982287936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,8192,0.04955200023121304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,6144,0.039650665389166936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,4096,0.010866666833559671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,3584,0.009992000129487779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,4096,0.030414223670959473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,3072,0.00928177767329746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,2560,0.01053244454993142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,5120,0.03527733352449205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,3584,0.028039998478359644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,65536,0.4758346875508626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,2048,0.009507555928495195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,1536,0.008021333151393467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,3072,0.025759999950726826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,1024,0.006153777655627992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,2560,0.023392889234754775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,768,0.005407111098368962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,2048,0.02096533278624217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,7168,0.04481955700450473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,512,0.004387555436955558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,1536,0.018441778090265062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,256,0.0035786665976047516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,1024,0.015600888265503777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,128,0.002781333401799202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,768,0.014555555250909595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,64,0.0021688888470331826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,256,0.012093333734406365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,768,32,0.002537777854336633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,512,0.013397333522637686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,64,0.010890666809346942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,32,0.010633777413103314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,16384,0.022258667482270136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,65536,0.1265582243601481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,12288,0.01556711064444648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,768,128,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,10240,0.013791999883121915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,8192,0.01185155577129788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,16384,0.08748710817760891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,12288,0.06842844353781806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,6144,0.011389333340856763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,7168,0.01129422254032559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,8192,0.049386666880713574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,10240,0.05875377522574531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,7168,0.044489777750439115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,5120,0.010029333333174387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,4096,0.008770667016506195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,3584,0.00829777783817715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,3072,0.007736889024575551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,65536,0.4590862062242296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,6144,0.03978666663169861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,5120,0.035227557023366295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,2560,0.0070186663005087115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,4096,0.0303439994653066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,2048,0.006392000036107169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,3584,0.02810844447877672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,1536,0.00904888908068339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,3072,0.025582222474945918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,1024,0.006684444430801604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,768,0.005492444253630108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,2560,0.023190221852726404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,2048,0.020812445216708712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,1536,0.018328888548745047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,512,0.004288889053795072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,256,0.003128888913326793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,128,0.002495999965402815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,64,0.0020702222569121257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,768,0.01443377799457974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,1024,0.015660444895426433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,512,0.013215111361609565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,512,32,0.0022684445397721394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,256,0.011925333076053195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,128,0.011160888605647616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,16384,0.013767999907334646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,65536,0.05486577749252319
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,64,0.010919111470381418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,512,32,0.010613333019945355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,12288,0.01147555559873581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,10240,0.010383110907342698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,8192,0.008729777402347988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,16384,0.08726311392254299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,12288,0.06824622551600139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,7168,0.008261333737108443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,10240,0.05875644418928358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,6144,0.007812444534566667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,5120,0.007024889190991719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,7168,0.04449244340260824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,8192,0.04941066768434313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,4096,0.006337777607970768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,3584,0.005992888990375731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,3072,0.006020444548792309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,6144,0.03978933228386773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,5120,0.03506755497720506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,2560,0.005464000006516774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,4096,0.030397332376903955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,3584,0.028088887532552082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,65536,0.4305733309851752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,2048,0.005028444445795483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,3072,0.025584000680181716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,1536,0.008774222599135505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,1024,0.0064444442590077715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,768,0.005261333452330695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,2560,0.02313511073589325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,2048,0.02071466710832384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,256,0.0029493332323100832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,512,0.004110222061475118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,1536,0.018389332625601027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,1024,0.015531554818153381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,128,0.0023680000255505242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,512,0.01310755560795466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,64,0.002230222233467632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,256,0.011930666863918304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,256,32,0.002179555594921112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,768,0.014524444937705994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,64,0.010854221880435944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,16384,0.009341333475377824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,65536,0.028095109595192805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,128,0.011150221857759686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,256,32,0.010604444477293225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,10240,0.0075146663520071245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,12288,0.00812888890504837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,8192,0.006680000159475539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,7168,0.009113777842786577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,16384,0.08743644422954983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,12288,0.06828444533877902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,10240,0.0588631100124783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,6144,0.006062222023804982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,5120,0.005649777750174205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,7168,0.04449422160784403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,8192,0.049363556835386485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,4096,0.005544888890451855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,3584,0.0069511110583941145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,6144,0.03969066672854953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,3072,0.006423111177153057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,5120,0.03516799873775906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,4096,0.03050755461057027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,2560,0.0058293334311909145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,3584,0.02798488736152649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,2048,0.005604444278611078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,1536,0.004679111142953237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,65536,0.4211306571960449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,3072,0.025607999828126695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,1024,0.005666666560702854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,2560,0.023286221755875483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,2048,0.020834666159417894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,768,0.0047324444684717385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,512,0.0037439999481042228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,1536,0.018228444788191054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,1024,0.015487111277050443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,256,0.002833777831660377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,128,0.002305777743458748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,64,0.0021520000365045336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,768,0.014346667461925082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,512,0.013215999636385175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,128,32,0.0021262222694026097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,256,0.011909333368142446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,65536,0.0237128883600235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,16384,0.00960799968904919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,64,0.010845333337783813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,128,0.011067555182509951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,12288,0.008353778057628209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,128,32,0.010704889065689512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,10240,0.007731555236710443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,8192,0.006953777538405524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,7168,0.007074666519959767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,12288,0.06830400228500366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,16384,0.08723555670844184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,10240,0.05844000313017103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,8192,0.04910133282343546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,6144,0.006481777876615524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,7168,0.04419555597835117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,5120,0.006000888844331105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,4096,0.0053582220441765254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,3584,0.0052684446175893145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,3072,0.005183111048407025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,6144,0.0396053327454461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,5120,0.03480888737572564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,65536,0.41277244355943465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,2560,0.0047395556337303585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,4096,0.030185778935750324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,3584,0.02781955401102702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,2048,0.00463377767139011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,1536,0.004037333445416557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,1024,0.005519999812046687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,3072,0.02530577778816223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,768,0.004573333180612988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,2560,0.023000889354281958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,512,0.003629333443111844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,1536,0.01812266641192966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,2048,0.020540444387329947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,1024,0.01536266671286689
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,256,0.0027022223091787765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,128,0.0022719999154408774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,768,0.01427111029624939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,64,0.0021555556191338436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,512,0.012905778156386482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,64,32,0.002130666747689247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,256,0.011640888949235281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,65536,0.014887111054526435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,128,0.011030221978823343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,16384,0.007382222347789341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,64,0.010773333410422007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,12288,0.00684799998998642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,64,32,0.010667555862002902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,10240,0.006120000034570694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,8192,0.006164444403515921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,7168,0.005528000079923206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,16384,0.08629688951704238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,6144,0.005504000104135937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,10240,0.058021333482530385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,12288,0.06730133295059204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,8192,0.04882133338186476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,5120,0.005071111023426056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,4096,0.004915555732117759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,7168,0.0440577765305837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,3584,0.004652444273233414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,3072,0.00434400017062823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,6144,0.0390897790590922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,5120,0.03459999958674113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,4096,0.02994222111172146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,65536,0.42515378528171116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,2560,0.004134222037262387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,3584,0.02755111124780443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,2048,0.0038337777886125776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,3072,0.025281776984532673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,1536,0.003768889026509391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,1024,0.0034933334423436057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,2560,0.022630222969584998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,768,0.004485333131419288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,2048,0.020435555113686454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,1536,0.01794755624400245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,512,0.003569777641031477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,256,0.00282577777074443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,1024,0.01536711057027181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,128,0.0022088889446523455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,768,0.014198222094111972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,64,0.0021315556433465746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,512,0.012840000291665396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,192,32,32,0.0020924444413847392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,256,0.011666666302416058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,128,0.01089866707722346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,64,0.010638222098350525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,192,32,32,0.01071733319097095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,10240,1.399338616265191
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,12288,1.6744862662421331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,8192,1.8041386074490016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,16384,2.1653974321153426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,10240,2.2050302293565536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,8192,1.0762355592515733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,7168,0.9072826173570422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,12288,2.8830568525526257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,6144,0.802959124247233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,5120,0.6205573081970215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,6144,1.384352895948622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,16384,3.4896827273898654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,5120,1.126307593451606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,7168,1.6669635772705078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,4096,0.5419164233737522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,4096,0.9119724697536893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,3584,0.7844933403862847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,3584,0.4759670893351237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,2560,0.5678266419304742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,2560,0.34341687626308864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,3072,0.4194088776906331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,2048,0.4957520167032878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,1536,0.23201065593295625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,3072,0.735786649915907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,1024,0.2540666792127821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,1536,0.35619910558064777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,1024,0.17707288265228271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,2048,0.3019164403279622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,256,0.06595822175343831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,512,0.1163911157184177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,768,0.14697778224945068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,768,0.20966933833228218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,512,0.1186337735917833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,32,0.018444443742434185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,128,0.10022222333484226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,64,0.09179555707507664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,128,0.03872266742918227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,32,0.09461422099007501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,65536,64,0.020202666521072388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,65536,256,0.10148088799582587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,16384,0.5952435599433051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,10240,0.40174399481879336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,12288,0.5012408892313639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,10240,0.5934515529208714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,16384,0.8676133155822754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,12288,0.7991369035508898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,8192,0.44377154774136013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,8192,0.3042808903588189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,7168,0.2601279947492811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,6144,0.21091200245751274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,7168,0.4186088773939345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,6144,0.3721342351701524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,5120,0.18220978313022188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,5120,0.2983448770311144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,3584,0.19762755764855278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,3584,0.12753600544399685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,4096,0.24883821275499132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,3072,0.17203288608127168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,3072,0.10971911085976495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,2560,0.13756267229715982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,4096,0.15222133530510798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,2560,0.09569778045018514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,1536,0.08168888754314847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,2048,0.12035732799106175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,768,0.04198399848408169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,65536,1.9979572296142578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,1024,0.06234577629301283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,2048,0.08154844575458102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,512,0.025601777765485976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,768,0.04456088940302531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,256,0.013049778011110095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,128,0.008362666600280339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,1024,0.056799113750457764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,512,0.039657778210110135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,64,0.006716444260544247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,256,0.03566933340496487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,32,0.004964444372389051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,1536,0.07224533292982313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,64,0.030894223186704848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,128,0.03290044599109226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,16384,32,0.031390223238203264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,16384,0.4611288706461589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,12288,0.5474257998996311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,16384,0.6723768975999621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,12288,0.3862408796946208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,16384,65536,3.395936965942383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,10240,0.29493334558275014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,10240,0.4140160083770752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,8192,0.36871732605828184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,7168,0.19852977328830293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,8192,0.2536542150709364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,6144,0.2875644365946452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,6144,0.16800710890028211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,65536,1.637380494011773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,7168,0.32038844956292045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,5120,0.2148817777633667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,5120,0.14797600110371908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,4096,0.17299021614922416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,4096,0.11374221907721625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,3072,0.12387200196584065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,3584,0.10804444551467896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,2048,0.08395910925335354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,2560,0.1033066643608941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,3584,0.1598400010002984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,3072,0.0968808862898085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,1536,0.06764088736640082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,2560,0.08086933030022515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,1024,0.05079022381040785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,2048,0.07323555813895331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,1536,0.05915111303329468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,512,0.017140444782045152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,768,0.03970844546953837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,768,0.03423644436730279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,1024,0.043988443083233304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,128,0.008008889026112026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,512,0.03597333365016513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,64,0.006184000107977126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,256,0.01163644426398807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,256,0.032369779215918645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,32,0.0041662222809261745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,128,0.029151111841201782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,12288,65536,2.5633341471354165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,32,0.0277528895272149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,12288,64,0.027832888894610937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,10240,0.3487866719563802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,12288,0.348673767513699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,10240,0.28502045737372506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,16384,0.4529866642422146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,12288,0.44365954399108887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,16384,0.6053831312391493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,7168,0.23367643356323242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,8192,0.2842826578352186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,8192,0.22771199544270834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,6144,0.2622533374362522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,6144,0.17039555973476836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,5120,0.21424533261193168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,4096,0.16773333814409044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,4096,0.11202400260501438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,7168,0.2012444469663832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,2560,0.08215644624498156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,3584,0.10097777843475342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,3584,0.15800978077782526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,5120,0.14227555857764349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,2048,0.06648533211814033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,1536,0.051345778836144336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,3072,0.10812444157070583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,2560,0.07917688952551948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,2048,0.07127377721998426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,1024,0.03587199913130866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,768,0.017836444907718234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,1024,0.04287999868392944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,512,0.013027555412716336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,3072,0.09528799851735432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,1536,0.055897778934902616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,768,0.03889866669972738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,128,0.006800888727108638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,65536,1.6205849117702906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,256,0.009139555195967356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,512,0.034782220919926964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,256,0.03125777840614319
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,64,0.005629333357016246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,128,0.028038223584493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,32,0.0037368887828456033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,32,0.02667733364635044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,10240,64,0.027311111489931744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,10240,65536,2.157080968221029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,16384,0.3203830983903673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,12288,0.35495466656155056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,10240,0.1977004475063748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,12288,0.24501689275105795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,16384,0.45798932181464297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,10240,0.304712004131741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,8192,0.2261573208702935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,7168,0.12237244182162815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,6144,0.1713493267695109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,5120,0.1322337786356608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,8192,0.1629777749379476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,5120,0.08839378092024062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,7168,0.21278399891323516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,6144,0.1127857764561971
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,4096,0.11969510714213054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,4096,0.07487466600206164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,3584,0.10456977950202094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,3072,0.09083733293745254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,65536,1.1155778037177193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,3584,0.06695022185643514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,3072,0.06111911270353529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,2048,0.061780446105533175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,1536,0.04232622186342875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,2560,0.06899199883143108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,2560,0.057421333260006376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,2048,0.045992887682384916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,768,0.015050666199790107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,1536,0.033048888047536217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,512,0.010961777634090848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,1024,0.02410488824049632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,1024,0.028816888729731243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,256,0.007256888680987888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,512,0.024642666180928547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,64,0.004440000073777305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,768,0.026741334133678015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,128,0.005324444423119227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,32,0.003293333368168937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,8192,65536,1.734159999423557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,64,0.02001333402262794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,128,0.0206977774699529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,32,0.019736889335844252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,8192,256,0.022413333257039387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,12288,0.21663199530707467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,12288,0.31140976481967503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,16384,0.2846862210167779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,10240,0.2678408887651232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,16384,0.42932621637980145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,8192,0.20008799764845106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,8192,0.127893328666687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,10240,0.17268799410925972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,7168,0.170270217789544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,7168,0.11576267083485921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,5120,0.11652889516618516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,6144,0.14642400211758083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,6144,0.09525244765811497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,65536,1.0133209228515625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,4096,0.10764888922373454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,3584,0.06160177787144979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,5120,0.08681777450773452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,2560,0.060686224036746554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,4096,0.06766311327616374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,3072,0.058803558349609375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,2048,0.04955288767814636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,2560,0.04929066697756449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,1536,0.04035377833578322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,1024,0.017106667160987854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,2048,0.0370142228073544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,3584,0.09264177746242946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,1536,0.031239112218221027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,768,0.013873777455753751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,3072,0.08095377683639526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,1024,0.027134221461084154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,512,0.012107555237081317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,65536,1.5264488855997722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,128,0.005560889012283749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,256,0.008808888494968414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,768,0.025109334124459162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,64,0.0038782221575578055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,512,0.023144889209005568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,7168,32,0.003118222165438864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,256,0.02164977788925171
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,64,0.018792000081804063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,128,0.020184000333150227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,7168,32,0.01872177753183577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,16384,0.33789245287577313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,12288,0.26849243375990123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,10240,0.20830933252970377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,12288,0.18315822548336455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,16384,0.2588355541229248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,10240,0.15149244997236463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,8192,0.17159999741448295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,7168,0.14233511024051243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,8192,0.10663733217451309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,7168,0.09269777933756511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,4096,0.08556711011462742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,6144,0.12411822213066949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,3584,0.07179199986987643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,5120,0.11382755968305801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,4096,0.0606328911251492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,6144,0.09285600317849053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,3072,0.06324177980422974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,65536,0.9094062381320529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,5120,0.07829244269265069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,3584,0.05717777543597751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,2048,0.04331733451949226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,3072,0.049957331683900624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,2560,0.05308977762858073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,1536,0.03170488940344916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,1024,0.01695999999841054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,768,0.013777777552604675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,512,0.010500444306267632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,2048,0.03420799970626831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,256,0.008065777520338694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,2560,0.04285866684383816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,1024,0.02607733342382643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,768,0.024040000306235418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,65536,1.3145582411024306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,128,0.005033777819739448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,512,0.022707555029127333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,1536,0.030631999174753826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,64,0.003620444486538569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,6144,32,0.0028897778441508612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,256,0.019855999284320407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,128,0.0185217774576611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,32,0.01882755590809716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,16384,0.290596432156033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,6144,64,0.018132444885041978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,12288,0.22219733397165933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,12288,0.14226222038269043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,16384,0.20888177553812662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,10240,0.18841421604156494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,8192,0.14377955595652261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,7168,0.12502843803829616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,8192,0.09360000160005356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,10240,0.11961155467563206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,7168,0.07990488741132948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,5120,0.0922071139017741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,65536,0.7966986762152778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,6144,0.11174133088853623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,5120,0.06790844599405925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,4096,0.07493244277106391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,6144,0.07204800181918673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,3584,0.05777244435416328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,2560,0.04195911023351881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,65536,1.1012532975938585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,4096,0.05376977721850077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,2048,0.034317334493001304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,3584,0.04842399888568454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,3072,0.04173688756095039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,3072,0.05610399776034885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,1536,0.01609777741962009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,1024,0.011561777856614856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,768,0.009546666509575313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,2048,0.03403733174006144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,512,0.007365333537260692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,1024,0.025352888637118872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,256,0.005163555757866966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,1536,0.02942399846182929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,128,0.004074666649103165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,2560,0.03798044390148587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,64,0.0033475555893447665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,768,0.024225778049892847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,512,0.021846221552954778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,5120,32,0.0031884445084465873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,64,0.01754044493039449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,32,0.017663111289342243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,128,0.018589332699775696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,5120,256,0.019567999574873183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,16384,0.1481448941760593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,12288,0.18285600344340006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,16384,0.2361057864295112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,8192,0.11086399687661065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,12288,0.10826133357153998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,10240,0.14145778285132513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,8192,0.07120711273617215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,7168,0.09889066881603664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,6144,0.08562933074103461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,10240,0.08613955312305027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,5120,0.07112799750434028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,65536,0.8860639995998807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,7168,0.0668026672469245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,65536,0.7234542104932996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,4096,0.038972444004482694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,5120,0.04980533321698507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,3584,0.051832000414530434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,6144,0.05702666441599528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,4096,0.058838221761915416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,2560,0.03373333480623033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,2048,0.01717333330048455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,1536,0.013956444131003486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,3584,0.03015466531117757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,3072,0.04575466778543261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,3072,0.028112000889248315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,2048,0.022708444131745234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,512,0.006593777901596493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,2560,0.025474665893448725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,1024,0.01795377830664317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,768,0.01720177796151903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,1024,0.010034666293197209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,1536,0.020399111840460036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,768,0.008443555898136562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,128,0.0041031113101376426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,64,0.003063999944263034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,512,0.015498666299713982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,256,0.004866666677925321
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,4096,32,0.0029031110720502008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,256,0.014365333649847241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,64,0.013014222184816996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,128,0.013192888763215808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,4096,32,0.012839111189047495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,12288,0.15351377593146429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,10240,0.12205155690511067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,12288,0.09410755501853095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,16384,0.21074755986531576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,10240,0.07971111271116468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,16384,0.13460532824198404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,7168,0.08774133523305257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,8192,0.10384888781441583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,8192,0.06566222508748372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,7168,0.05848711066775852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,5120,0.06529155704710218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,6144,0.0520631108019087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,6144,0.0761715571085612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,4096,0.053704000181621976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,3584,0.04630577895376417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,65536,0.6095253096686469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,65536,0.7849813567267524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,4096,0.031903111272388034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,3072,0.03537955549028184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,3584,0.02970044480429755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,2048,0.016746666696336534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,5120,0.045732445187038846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,2560,0.02942488922013177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,3072,0.027319999204741582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,1536,0.013376889129479727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,1024,0.009910222556855943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,768,0.008281778130266402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,512,0.0064995553758409284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,2048,0.022494221727053326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,1536,0.02010933392577701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,256,0.004837333328194088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,768,0.016727111405796476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,128,0.0038515557017591265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,512,0.01535377734237247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,2560,0.025128000312381323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,1024,0.017584888471497428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,64,0.003036444385846456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3584,32,0.002765333279967308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,128,0.013028444515334235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,64,0.012896888785892062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,256,0.013945778210957846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3584,32,0.012818666795889536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,16384,0.18416799439324272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,12288,0.13081955909729004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,16384,0.11318400171067979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,12288,0.08519288566377427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,8192,0.0843813353114658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,10240,0.0731031099955241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,10240,0.10636711120605469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,6144,0.0675813357035319
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,7168,0.07797600163353814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,7168,0.054217778974109225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,8192,0.061158219973246254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,5120,0.05710133579042223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,65536,0.5610799789428711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,4096,0.04719110992219713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,6144,0.04833955566088358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,3072,0.030976888206270006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,3584,0.04009600149260627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,65536,0.6841360198126899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,5120,0.03829511006673177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,4096,0.031641778018739485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,2560,0.02011644509103563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,2048,0.016708445217874315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,1536,0.013178666432698568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,3072,0.026735110415352717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,1024,0.00981955561372969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,2560,0.02460266649723053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,2048,0.022090666823916968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,768,0.00813422186507119
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,1536,0.01975022256374359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,512,0.0063982220987478895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,256,0.004676444249020683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,3584,0.0295724438296424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,128,0.0035911111368073356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,1024,0.017543110582563613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,256,0.013759111364682516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,64,0.00282577777074443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,3072,32,0.0026559999419583213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,512,0.014907555447684394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,128,0.012679999901188744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,64,0.012531555361217923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,768,0.01630577776167128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,3072,32,0.012526222401195101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,16384,0.14398045010036892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,12288,0.10518311129675971
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,10240,0.08660355541441177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,16384,0.10310844580332439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,8192,0.07106755839453803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,12288,0.08435644706090291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,8192,0.056107554170820445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,10240,0.06758844190173678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,7168,0.06266844272613525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,7168,0.049490667051739164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,6144,0.055380443731943764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,65536,0.5800995296902126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,6144,0.039974222580591835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,3584,0.0343413319852617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,5120,0.04705244302749634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,4096,0.042965332667032875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,65536,0.5264586872524685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,4096,0.030551112360424463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,3072,0.018617777360810172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,5120,0.0351235568523407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,2560,0.01665155589580536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,1536,0.010959111154079437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,2048,0.014540443817774454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,3584,0.028248889578713313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,3072,0.025898666845427618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,1024,0.008169777691364288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,2048,0.021236444513003033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,512,0.005511999958091312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,1536,0.018944889307022095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,256,0.0047315553658538395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,2560,0.023629332582155865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,768,0.015383111106024848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,128,0.0033573332346147964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,768,0.006849777781301075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,512,0.014016888207859464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,64,0.0027679999669392905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,1024,0.016383111476898193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2560,32,0.0025315555847353404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,64,0.011732444167137146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,32,0.011768889096048145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,256,0.013001778059535556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2560,128,0.011832888755533429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,12288,0.08311377631293403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,10240,0.0704782207806905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,16384,0.10788266526328193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,16384,0.09336266915003459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,12288,0.07043466965357463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,8192,0.05783466498057047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,10240,0.06181155310736763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,7168,0.05115466647677951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,65536,0.4784835709465875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,5120,0.03859555390146043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,6144,0.04498933421240913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,4096,0.024534222152498033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,8192,0.05146400133768717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,6144,0.039015998442967735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,5120,0.03455288873778449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,65536,0.4984284506903754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,3584,0.017624000708262127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,3072,0.016738666428460013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,7168,0.04366044534577263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,2048,0.013487111363146039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,3584,0.02769688930776384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,2560,0.015271999769740634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,1536,0.009322666459613377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,2560,0.02323466704951392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,4096,0.029983109898037378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,2048,0.020985777179400127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,1536,0.01826755536927117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,768,0.005997333261701796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,1024,0.007117333511511485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,512,0.004955555415815777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,3072,0.025450666745503742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,1024,0.015961777832772996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,128,0.003173333282272021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,64,0.002571555475393931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,768,0.014874666929244995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,32,0.0023893333143658107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,512,0.013611555927329592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,256,0.01238933371173011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,128,0.011392888923486074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,64,0.011380444798204633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,2048,256,0.004530666602982415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,2048,32,0.011189333266682096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,16384,0.0827982227007548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,12288,0.06394844584994845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,10240,0.05450310972001818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,16384,0.086535996860928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,12288,0.06629688872231378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,8192,0.04505777690145704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,7168,0.0376142230298784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,65536,0.36551909976535374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,8192,0.04771999849213494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,10240,0.05751466751098633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,5120,0.01850666602452596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,7168,0.04331022169854906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,6144,0.038867556386523776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,6144,0.03287022312482198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,4096,0.015569777952300178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,3584,0.0144195556640625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,65536,0.46367465125189883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,3072,0.013039111263222165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,2560,0.012092444631788464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,4096,0.029564443561765883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,5120,0.034288889831966825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,3584,0.02760266595416599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,2048,0.0158915552828047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,1536,0.013187555803192986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,3072,0.025207999679777358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,2560,0.022818666365411546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,1024,0.009530666801664565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,768,0.007872000336647034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,2048,0.020615110794703167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,512,0.00600355573826366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,256,0.004371555729044808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,1024,0.015659555792808533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,128,0.0033191111352708605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,1536,0.018034666776657104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,768,0.014686221877733866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,512,0.013527111046844058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,32,0.0023306666149033443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1536,64,0.002624888800912433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,256,0.012248000337017907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,128,0.01108266661564509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,64,0.010873777998818291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1536,32,0.010791111323568555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,16384,0.05727644761403402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,12288,0.04474755459361606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,10240,0.03589955634540982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,8192,0.018578666779730055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,16384,0.08430222007963394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,12288,0.06578310992982653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,65536,0.25471554862128365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,7168,0.017310221989949543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,6144,0.015673778123325773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,10240,0.05701155794991387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,5120,0.013562666873137156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,8192,0.04773955543835958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,7168,0.043190220991770424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,6144,0.0385484430525038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,3584,0.011065777805116443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,4096,0.011618666350841522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,3072,0.010284444524182213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,5120,0.03417155477735732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,2560,0.009307555854320526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,2048,0.009646221995353699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,65536,0.4199688964419895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,3584,0.02747111188040839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,1536,0.008015111088752747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,3072,0.025078222155570984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,1024,0.006218666831652324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,4096,0.029633777009116277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,2560,0.022760889596409265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,2048,0.02040088839001126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,768,0.005429333282841577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,512,0.004414222306675381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,256,0.003519999898142285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,1536,0.01789422167672051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,1024,0.015412444869677225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,128,0.002855111120475663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,768,0.014589332871966891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,64,0.0022177776942650476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,512,0.013165333204799227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,1024,32,0.002136888810329967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,256,0.012062222593360476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,128,0.011152000062995486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,64,0.010847999817795224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,1024,32,0.01088444474670622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,10240,0.01759822169939677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,12288,0.03306488858328925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,12288,0.06568800078497992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,16384,0.04485955503251818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,65536,0.19840444458855522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,8192,0.014919110470347934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,16384,0.08421066734525894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,7168,0.013955555028385587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,10240,0.057130667898390025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,6144,0.012682666381200155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,5120,0.011316444310877057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,8192,0.04760266674889458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,4096,0.010789333118332757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,65536,0.40423199865553117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,3584,0.010117332968446944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,7168,0.043135109874937266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,6144,0.038648890124426946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,5120,0.034302221404181585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,3072,0.009140444298585257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,2560,0.008332444561852349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,4096,0.02961955467859904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,2048,0.011523555550310345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,3584,0.02731200059254964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,1024,0.006787555499209299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,3072,0.02495555579662323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,2560,0.022849778334299724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,2048,0.020303999384244282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,768,0.00564533347884814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,1536,0.017952000101407368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,1536,0.009203555683294931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,512,0.004389333228270213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,256,0.0032631111227803757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,128,0.002620444529586368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,1024,0.015536889433860779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,64,0.0021377777059872947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,768,0.014363555444611443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,768,32,0.0023377777801619633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,512,0.013159111142158508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,256,0.012005333271291522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,128,0.011139555937714048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,16384,0.025847999585999384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,64,0.010904889139864178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,768,32,0.010856889188289642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,65536,0.11563111676110162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,12288,0.015053333507643806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,10240,0.013261333107948303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,8192,0.011425777441925473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,7168,0.010945777926180096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,16384,0.08396355311075847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,12288,0.0657964415020413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,10240,0.05704355239868164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,6144,0.010879110958841113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,5120,0.009619555539555019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,8192,0.0476453337404463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,7168,0.04299733373853895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,4096,0.008359111017651027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,3584,0.00794311116139094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,6144,0.038584887981414795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,5120,0.03416000141037835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,3072,0.0074106670088238185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,65536,0.3880755636427138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,2560,0.0068808889223469635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,3584,0.027262222435739305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,4096,0.02951288885540432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,3072,0.02493422230084737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,1536,0.008955555657545725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,2560,0.022600889205932617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,1024,0.006610666712125142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,2048,0.006238222122192383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,2048,0.020264888803164165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,768,0.005442666510740916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,1536,0.017870222528775532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,512,0.004265777766704559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,1024,0.01531733406914605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,256,0.003116444374124209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,128,0.00249777775671747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,64,0.0020879999630981022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,512,32,0.002275555498070187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,768,0.014398222168286642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,512,0.01312444441848331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,256,0.011838222543398539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,128,0.011104889214038849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,64,0.01093777765830358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,16384,0.013156444662147097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,12288,0.010420444111029306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,512,32,0.010696888797812991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,65536,0.05491911040412056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,10240,0.009527111219035255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,8192,0.00851466672288047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,7168,0.00797155582242542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,16384,0.08387466933992173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,12288,0.06592000193066068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,6144,0.0074782222509384155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,10240,0.05693333016501533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,8192,0.04766755633884006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,5120,0.006776888751321369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,7168,0.04302488764127096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,4096,0.006172444257471297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,3584,0.0059368887709246735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,3072,0.005653333332803514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,6144,0.038527111212412514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,5120,0.03420266509056091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,2560,0.005422222117582957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,65536,0.35444000032212997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,3584,0.027202667461501226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,4096,0.029559112257427637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,2048,0.00499555551343494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,1536,0.005055111315515307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,3072,0.02498488956027561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,2560,0.022594667143291895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,2048,0.020181333025296528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,1024,0.0064328888224230866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,768,0.0052435555391841466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,512,0.004047111090686587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,256,0.002944888960984018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,1536,0.0177528891298506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,128,0.002324444345302052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,1024,0.01534577707449595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,768,0.014372444815105863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,512,0.013089777694808112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,64,0.0021991110924217436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,256,32,0.002171555534005165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,256,0.011921777493423887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,128,0.011123555401961008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,16384,0.009143999881214565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,64,0.0107342220014996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,65536,0.03731377919514974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,12288,0.010823110739390055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,256,32,0.010861333045694562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,10240,0.00979644474056032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,8192,0.00848444468445248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,7168,0.008287111090289222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,16384,0.08404711219999526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,6144,0.007689778175618913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,10240,0.05693066782421536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,12288,0.06598133511013456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,8192,0.04766755633884006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,5120,0.007095110913117726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,7168,0.04318755533960131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,4096,0.006616888774765863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,3584,0.006035555568006303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,65536,0.34891022576226133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,6144,0.03865866528617011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,3072,0.0058648888435628675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,5120,0.034049779176712036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,2560,0.005420444326268301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,2048,0.0052177777720822235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,4096,0.02955199943648444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,3072,0.024808888634045918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,1536,0.004504000147183736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,2560,0.022760889596409265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,1024,0.005659555395444234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,768,0.0046693334976832075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,3584,0.027327110370000202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,2048,0.02015111181471083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,512,0.00370755543311437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,1536,0.0177831103404363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,256,0.002817777709828483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,1024,0.015209777487648858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,768,0.014363555444611443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,128,0.0022942223068740633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,64,0.002141333288616604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,512,0.013168888787428537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,128,32,0.0021191111041439907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,256,0.011865778101815118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,128,0.011047999891969891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,64,0.010778666370444827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,65536,0.020488889680968392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,16384,0.008757332960764566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,128,32,0.010582221878899468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,12288,0.007673777639865875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,10240,0.007256888680987888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,8192,0.006552888701359431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,7168,0.006230222268237009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,12288,0.06566133101781209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,10240,0.05662666426764595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,6144,0.006085333310895496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,8192,0.047338667843076915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,16384,0.083697775999705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,7168,0.04267911116282145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,5120,0.005550222264395819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,4096,0.005241777747869492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,3584,0.004958222309748332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,3072,0.004678222040335338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,6144,0.038463112380769514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,5120,0.03391022152370877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,65536,0.3452586597866482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,2560,0.004819555415047539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,4096,0.029270221789677937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,3584,0.026872888207435608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,2048,0.004551110996140374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,1536,0.003919110943873723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,3072,0.024622221787770588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,1024,0.003628444340493944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,768,0.004549333204825719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,2560,0.022484444909625586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,512,0.0036231109665499795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,256,0.0026968889352348116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,2048,0.020042666130595736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,1536,0.017567111386193168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,1024,0.015230221880806817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,768,0.01400888959566752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,128,0.002248888835310936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,64,0.002151111140847206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,512,0.012906666431162091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,64,32,0.0021155555215146807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,256,0.01163999984661738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,65536,0.01638488968213399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,16384,0.0070062221752272705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,64,0.010784000158309937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,128,0.01091822236776352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,64,32,0.010662222074137794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,12288,0.005986666513813867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,10240,0.006112888869312074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,8192,0.005507555686765247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,7168,0.005003555367390315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,16384,0.08402578035990398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,6144,0.004770666774776247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,12288,0.06574577755398221
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,10240,0.05701422029071384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,5120,0.004948444250557157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,8192,0.047520889176262744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,4096,0.004784000002675586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,7168,0.042746666404936046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,3584,0.004363555461168289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,3072,0.0041519999504089355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,6144,0.038352890147103205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,5120,0.03386933273739285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,2560,0.004018666843573253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,65536,0.34729690021938747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,4096,0.029313776228162978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,2048,0.003825777934657203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,1536,0.0037004442678557504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,1024,0.0034799998005231223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,3584,0.027056889401541814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,3072,0.02459022237194909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,2560,0.022252443763944838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,2048,0.01994311147265964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,768,0.004477333277463913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,512,0.003563555578390757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,1536,0.01750666730933719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,256,0.0028115556471877625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,128,0.002206222257680363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,1024,0.015073777900801765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,64,0.0021235555824306277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,768,0.014058666096793281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,512,0.0127688886390792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,160,32,32,0.0020968889196713767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,256,0.01147377739350001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,128,0.01089244418674045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,64,0.010813333094120026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,160,32,32,0.010669333239396414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,10240,1.304449823167589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,12288,1.593123541937934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,10240,2.0601537492540145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,16384,2.0750969780815973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,8192,1.6712142096625433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,8192,0.9972062640719943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,12288,2.728135214911567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,6144,0.6982364654541016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,7168,0.8327937655978732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,7168,1.456559075249566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,16384,3.2657992045084634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,5120,0.6195511288113064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,6144,1.2942079967922635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,5120,1.0555280049641926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,4096,0.8544266488817004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,3584,0.4140462345547146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,4096,0.4997466405232747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,3584,0.7534213595920138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,2560,0.3096071084340413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,3072,0.36666756206088597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,2560,0.5510213110182021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,3072,0.7206480238172742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,1536,0.37232266532050234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,768,0.1910631126827664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,2048,0.4878560172186957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,2048,0.2828897900051541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,1024,0.2568257649739583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,1024,0.16701243983374703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,512,0.11767466862996419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,768,0.13277599546644422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,64,0.01238844460911221
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,128,0.03434755404790243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,1536,0.22506221135457358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,256,0.0585191117392646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,256,0.10066488716337417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,128,0.10393777820799087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,64,0.0986293355623881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,32,0.09897689024607341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,65536,32,0.014206222361988492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,65536,512,0.10598488648732503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,16384,0.5268844498528374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,10240,0.3929244412316217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,12288,0.42427910698784727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,10240,0.5804497930738661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,12288,0.7910995483398438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,16384,0.9486764272054037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,7168,0.37204888131883407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,8192,0.4416364563835992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,8192,0.31752800941467285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,7168,0.25282221370273167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,6144,0.3397253354390462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,5120,0.2700666586558024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,5120,0.17831644746992323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,6144,0.23606843418545195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,4096,0.239428440729777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,3584,0.19167821937137178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,4096,0.14446755250295004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,3072,0.16308532820807561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,2560,0.13188000520070395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,3072,0.10692889160580105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,2048,0.10596355464723374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,3584,0.12369599607255723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,1536,0.08001599709192912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,1024,0.05498399999406603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,65536,1.985802756415473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,768,0.04182755615976122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,1536,0.057625777191585965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,2560,0.09198133150736491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,2048,0.07698577642440796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,256,0.0075431110130416016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,128,0.005952888892756567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,1024,0.04450400008095635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,768,0.029099557134840224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,64,0.005075555708673266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,256,0.022553778356975977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,128,0.02040799955526988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,32,0.004299555387761858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,64,0.019920888874265883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,512,0.022879110442267522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,32,0.01981599960062239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,16384,512,0.025378665990299646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,10240,0.4117537869347467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,12288,0.3335084385342068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,16384,0.4172453350490994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,16384,0.6506746610005697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,12288,0.5863111284044054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,16384,65536,3.3686790466308594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,8192,0.3306266731686062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,10240,0.2730160024431017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,8192,0.21466488308376738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,7168,0.28907820913526744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,7168,0.19960444503360322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,6144,0.2580213281843397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,5120,0.22851822111341688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,5120,0.1304266717698839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,6144,0.16413954893747965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,65536,1.4784453709920247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,4096,0.18300088246663412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,3584,0.08770133389366998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,3072,0.11914043956332737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,4096,0.10615822341707017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,3072,0.07730488644705878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,2048,0.08192355765236749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,2560,0.07305866479873657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,3584,0.15737067328559026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,1536,0.061808890766567655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,2560,0.11236088805728489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,768,0.03254577848646376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,2048,0.060749332110087075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,1024,0.042596442831887134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,1024,0.02712533374627431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,256,0.007157333195209503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,1536,0.04675555560323927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,768,0.02426133387618595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,512,0.010859555668301053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,512,0.02221333318286472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,64,0.004659555438492033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,128,0.005286222116814719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,256,0.020657777786254883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,32,0.003773333297835456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,128,0.01847999956872728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,64,0.018058665924602084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,12288,32,0.018254222141371835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,12288,65536,2.488768047756619
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,16384,0.35783910751342773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,12288,0.4395875665876601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,16384,0.5475369029574925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,10240,0.3447431193457709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,12288,0.2860933409796821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,10240,0.22619822290208605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,8192,0.17409688896603057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,7168,0.14325422710842556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,7168,0.31672000885009766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,6144,0.2636400063832601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,5120,0.20658311578962538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,4096,0.14101866881052652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,6144,0.12970756159888372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,8192,0.3097519874572754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,5120,0.1099582248263889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,4096,0.08334400256474812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,3584,0.07442844576305814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,3072,0.10737244288126628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,65536,1.3912657631768122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,2560,0.08915999862882827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,3072,0.07251555389828153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,3584,0.13138311439090306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,2048,0.06972889105478923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,1024,0.03566755519972907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,2560,0.057968888017866343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,768,0.016505777835845947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,1536,0.052750223212771945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,2048,0.053780443138546415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,1024,0.025759999950726826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,512,0.01090755561987559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,1536,0.038450668255488075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,256,0.007111999723646376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,128,0.0052391112678580815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,512,0.021423111359278362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,65536,2.077869415283203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,768,0.023572444915771484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,32,0.003277333246337043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,128,0.018183110488785636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,64,0.017621333400408428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,10240,64,0.004499555461936527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,32,0.018232888645595975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,10240,256,0.01958933307064904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,10240,0.17430578337775338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,12288,0.23810489972432455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,12288,0.3543128967285156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,16384,0.4246728950076633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,16384,0.3197022279103597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,10240,0.29425154791937935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,8192,0.21951288647121855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,7168,0.19258311059739855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,8192,0.13769600126478407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,7168,0.10846044619878133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,6144,0.16892533832126191
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,6144,0.1024399995803833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,3584,0.09108444717195298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,4096,0.11659467220306396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,5120,0.1376311116748386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,3072,0.07835288842519124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,4096,0.07270666625764635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,2560,0.06611822048823039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,3072,0.053103112512164645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,3584,0.06522133615281847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,1536,0.04090755515628391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,2560,0.04614488946066963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,5120,0.08708977699279785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,65536,1.0414142608642578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,2048,0.053800000084771045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,2048,0.037068443165885076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,512,0.007845333053006066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,1024,0.01608533329433865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,768,0.010077333284748925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,256,0.005581333405441708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,1024,0.019935111204783123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,768,0.018704000446531508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,128,0.004335999902751711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,64,0.003666666646798452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,1536,0.022429333792792425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,256,0.015535111228624979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,65536,1.6740666495429144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,8192,32,0.002942222274012036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,64,0.014078223043017916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,128,0.013995556367768182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,512,0.016840888394249808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,8192,32,0.014022222823566861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,16384,0.256785790125529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,12288,0.18652710649702284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,12288,0.3126444551679823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,10240,0.2607955667707655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,16384,0.41663644048902726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,10240,0.14355377356211343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,8192,0.19504266315036348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,6144,0.14143910672929552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,7168,0.16429689195421007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,8192,0.106532441245185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,7168,0.10314133432176377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,5120,0.11333955658806695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,4096,0.09172978003819783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,5120,0.07120177480909559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,6144,0.08564444382985432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,3584,0.07968978087107341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,3072,0.0690480007065667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,2560,0.05781866444481743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,3584,0.053123556905322604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,4096,0.06432533264160156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,65536,1.0317271550496419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,3072,0.046816888782713145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,2048,0.04725422130690681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,1536,0.03618489040268792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,1024,0.011726222104496427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,768,0.009678222239017487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,65536,1.4706960254245336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,512,0.007855110698276097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,1024,0.018959111637539335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,1536,0.021428444319301184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,256,0.005498666730191972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,2560,0.04261422157287598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,2048,0.02604444490538703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,128,0.0041706665522522396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,64,0.003516444315512975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,768,0.01755466726091173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,7168,32,0.0033111110743549135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,256,0.015275556180212231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,128,0.013752889302041797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,64,0.013276444541083442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,32,0.013372444444232516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,7168,512,0.016327111257447135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,16384,0.21273599730597603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,16384,0.339031113518609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,12288,0.2727999952104357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,12288,0.14764622847239176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,8192,0.16544000307718912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,10240,0.11355733871459961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,7168,0.14006843831804064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,10240,0.23235556814405653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,8192,0.09065866470336914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,7168,0.07995200157165527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,6144,0.1219351159201728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,6144,0.07097778055402967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,65536,0.8202213181389703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,5120,0.11212889353434245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,4096,0.0793351133664449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,3584,0.0688391129175822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,4096,0.051206221183141075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,3072,0.05987199809816149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,5120,0.066666669315762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,2560,0.05004622207747566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,2048,0.040624890062544085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,1024,0.0107022225856781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,3072,0.044756445619795054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,2048,0.022473777333895367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,1536,0.03502577874395583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,1536,0.020136000381575692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,65536,1.2724622090657551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,2560,0.0299253331290351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,1024,0.018000000052981906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,256,0.005149333427349727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,128,0.003931555483076307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,512,0.006953777538405524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,256,0.01424088908566369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,768,0.00888355573018392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,64,0.0032337777730491427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,768,0.016781333420011733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,6144,32,0.003112888791494899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,3584,0.05004355642530653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,512,0.015403555499182807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,64,0.013047999805874295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,128,0.013349333571063148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,6144,32,0.013027555412716336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,12288,0.21632799837324354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,12288,0.11771289507548015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,16384,0.18047732777065703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,10240,0.0970497793621487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,16384,0.27319110764397514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,10240,0.18971999486287436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,8192,0.13987910747528076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,7168,0.13725244998931885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,8192,0.08089511262045966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,6144,0.10184888707266913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,7168,0.07103111346562703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,5120,0.09477688868840535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,5120,0.055128888951407544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,6144,0.06842133071687487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,4096,0.04602577620082431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,65536,0.7022364404466418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,4096,0.07911022504170735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,3584,0.06253511375851102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,2560,0.04179733329349094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,2048,0.03397511111365424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,3072,0.05565244621700711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,3584,0.04048711061477661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,65536,1.053617795308431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,3072,0.030082666211658057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,2560,0.0243857784403695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,1536,0.01368533323208491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,1024,0.010173333187898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,2048,0.02193333374129401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,768,0.008597333398130205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,512,0.006677333265542984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,256,0.005127111242877113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,128,0.0038177776667806837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,512,0.0151573336786694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,64,0.0031431110368834603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,256,0.013835555149449242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,5120,32,0.0028888889484935333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,1024,0.017557332913080852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,1536,0.019541333119074505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,768,0.016335111525323655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,128,0.01295111080010732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,64,0.012846222354306115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,5120,32,0.012628444366984896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,10240,0.1319777833090888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,12288,0.0997884472211202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,12288,0.17917244964175752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,16384,0.13527466191185847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,16384,0.2278613249460856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,7168,0.0931875573264228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,8192,0.10618933041890462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,10240,0.08442666795518662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,7168,0.06159377760357327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,8192,0.0696444445186191
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,5120,0.0698088871108161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,4096,0.056178669134775795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,6144,0.08251911401748657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,4096,0.03752800159984165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,65536,0.6069608794318305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,3584,0.04972889026006063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,6144,0.055052445994483105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,65536,0.8503164185418023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,3072,0.044066667556762695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,2048,0.015227556228637695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,5120,0.048584000931845776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,3584,0.027809778849283855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,2560,0.03254577848646376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,1024,0.008950221869680617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,1536,0.012096888489193387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,3072,0.025617778301239014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,2560,0.023299554983774822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,768,0.007461333440409766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,1536,0.018647111124462552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,256,0.004567111117972268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,512,0.005933333188295364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,1024,0.016522667474216886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,512,0.014194667339324951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,128,0.0035911111368073356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,64,0.0028826666788922418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,256,0.013010666602187686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,4096,32,0.0027173333283927706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,128,0.01203555530971951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,768,0.01555022266176012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,64,0.011756444142924415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,32,0.011713777979214987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,4096,2048,0.02102311121092902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,16384,0.19113777743445504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,12288,0.15239466561211482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,12288,0.090865777598487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,16384,0.12606133355034724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,8192,0.1062320007218255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,10240,0.07751822471618652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,10240,0.14187822077009413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,7168,0.08420799838172065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,6144,0.07336444324917264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,8192,0.06411200099521212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,4096,0.049955556790033974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,5120,0.07008444600635104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,6144,0.05097066693835788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,5120,0.045122666491402515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,7168,0.05688889159096611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,3584,0.04423911041683621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,3072,0.033655110332700945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,65536,0.560073799557156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,65536,0.7587324248419868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,2560,0.025366221865018208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,2048,0.01368533323208491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,1536,0.011214222345087262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,2560,0.023016000787417095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,1024,0.008464888566070134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,3584,0.02755111124780443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,4096,0.02979822291268243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,2048,0.02069422271516588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,1536,0.01847022275129954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,768,0.007103111180994246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,3072,0.02529955572552151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,512,0.0057795556883017225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,256,0.004382222063011593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,1024,0.01611911091539595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,128,0.003491555651028951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,768,0.015060444672902426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,512,0.013929777675204806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,32,0.002564444517095884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3584,64,0.002761777697337998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,256,0.012647111382749347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,128,0.011752888560295105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,32,0.011456888582971362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,16384,0.16547821627722845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,12288,0.12202844354841445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,16384,0.10769155290391709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3584,64,0.011527111132939657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,12288,0.08308266931109957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,8192,0.08363111151589288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,10240,0.10230133268568252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,10240,0.07055733601252238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,7168,0.0731911129421658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,6144,0.06418310933642916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,7168,0.05236355463663737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,5120,0.05485066771507263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,8192,0.06334755818049113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,4096,0.043423109584384494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,6144,0.046893331739637584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,3584,0.038768000072903104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,65536,0.5099280145433214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,4096,0.0292871097723643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,3584,0.027110222313139174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,3072,0.02909066610866123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,2560,0.016128000285890367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,5120,0.03496355480617947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,3072,0.024868445263968572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,1536,0.010301333334710863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,2048,0.01254488858911726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,65536,0.6525742212931315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,1024,0.007704888780911763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,768,0.006481777876615524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,2560,0.02272533377011617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,512,0.005380444642570283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,2048,0.020482665962643094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,256,0.004852444347408083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,1536,0.01796888808409373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,1024,0.015807110402319167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,128,0.0032915555768542816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,512,0.013592888911565145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,768,0.014739554789331226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,256,0.012455999851226807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,32,0.002471999989615546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,128,0.011315555208259158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,64,0.01124533348613315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,3072,32,0.011061333119869232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,3072,64,0.002736888825893402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,12288,0.09951644473605686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,16384,0.13240799638960096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,16384,0.09831644429100884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,10240,0.08471200201246475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,12288,0.07455644342634413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,8192,0.06862577464845446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,7168,0.060679111215803355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,10240,0.0651528901524014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,7168,0.04749866657786899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,8192,0.05377866824467977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,5120,0.044885334041383534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,6144,0.05247644583384196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,65536,0.5451093249850804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,4096,0.0370488895310296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,6144,0.038367112477620445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,3072,0.01688888834582435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,65536,0.4811849064297146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,5120,0.03345866666899787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,4096,0.02905422117975023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,2048,0.010772444307804108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,2560,0.013657777673668332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,1536,0.009176000124878353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,1024,0.007017778025733099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,2560,0.022669333550665114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,3584,0.03176888823509216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,2048,0.02016533414522807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,768,0.006052444378534953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,512,0.005121777868933148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,1536,0.017887110511461895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,3584,0.02700800034734938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,256,0.004000888930426704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,1024,0.01569600072171953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,3072,0.02475555572244856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,128,0.0032115555885765287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,64,0.0026151111556424033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2560,32,0.0023644444429212147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,512,0.013431111143694984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,256,0.012237333589129977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,64,0.011021333436171213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,128,0.011168888873524137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,32,0.01089333328935835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2560,768,0.014850666125615438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,12288,0.08011821905771892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,16384,0.10492089059617783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,10240,0.06764888763427734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,8192,0.0555004444387224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,12288,0.06808888912200928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,10240,0.058677335580190025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,16384,0.09087022145589192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,7168,0.048968887991375394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,6144,0.042725334564844765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,65536,0.4409484333462185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,7168,0.04193688763512505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,8192,0.04893866512510511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,5120,0.03650666607750787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,4096,0.018180444836616516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,6144,0.03764266769091288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,3584,0.013613333304723104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,65536,0.4558986557854547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,3072,0.01257599973016315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,5120,0.03310488992267185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,2560,0.021317332983016968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,4096,0.02888444397184584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,3584,0.026628444592158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,3072,0.024533333049880132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,1024,0.009691555466916826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,1536,0.0136817776494556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,2048,0.016208888755904306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,2048,0.019754666421148512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,2560,0.022256000174416438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,1536,0.01771288944615258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,768,0.008150222400824228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,256,0.004450666821665234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,512,0.004666666603750653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,128,0.003499555504984326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,1024,0.015449777245521545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,512,0.013025778035322825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,256,0.011961778004964193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,64,0.0026293332791990707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,2048,32,0.0023448889454205832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,768,0.0143857780430052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,128,0.01109688894616233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,64,0.010803555448849997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,2048,32,0.010671111444632212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,16384,0.08064622349209256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,12288,0.0619608892334832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,10240,0.052128000391854175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,16384,0.08227822515699598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,12288,0.06435110833909777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,8192,0.04288622074657016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,7168,0.03813866774241129
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,6144,0.03336711062325372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,65536,0.3406497902340359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,8192,0.04607377780808342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,10240,0.055380443731943764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,7168,0.04173333446184794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,4096,0.01239911135700014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,6144,0.03762311074468825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,3584,0.011546666423479715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,65536,0.42053866386413574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,3072,0.01071911139620675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,5120,0.033117334047953285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,4096,0.028767999675538804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,5120,0.01385244478782018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,3584,0.026664889521068994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,2048,0.00981244444847107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,3072,0.024404444628291663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,1536,0.008160889148712158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,1024,0.006342222293217977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,2560,0.02222577730814616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,768,0.005579555614127054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,2560,0.011720889144473605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,512,0.004599110947714912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,2048,0.019857777489556205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,256,0.0037048889531029593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,1024,0.01531377765867445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,768,0.01420177850458357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,1536,0.01756888959142897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,128,0.0030702222138643265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,64,0.0022986665782001284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,512,0.01314222233162986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1536,32,0.0022071111533376905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,256,0.011917333636018964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,128,0.011160000330872007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,64,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,16384,0.054811557133992515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1536,32,0.010729778144094678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,12288,0.04251111216015286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,16384,0.08084533611933391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,10240,0.0362408889664544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,8192,0.016953777935769822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,65536,0.24357867240905762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,7168,0.015142222245534262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,12288,0.06362311045328777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,6144,0.013727111121018728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,10240,0.05483555462625292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,5120,0.012072888513406118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,4096,0.010024888647927178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,7168,0.04197066691186693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,8192,0.046128001477983266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,6144,0.037581334511439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,3584,0.00941777808798684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,5120,0.033109333780076765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,65536,0.36945777469211155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,4096,0.028619554307725694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,2560,0.014119999276267158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,3072,0.008765333228641087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,1536,0.009320889082219865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,2048,0.011549333731333414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,3072,0.024440889557202656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,3584,0.026758222116364375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,2560,0.022082666556040447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,2048,0.019676445258988273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,768,0.005670222143332164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,1536,0.01757600075668759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,1024,0.006811555474996567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,512,0.00444533344772127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,256,0.0033440000067154565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,1024,0.015307555596033732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,128,0.0026373333401150177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,64,0.0021911110315057966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,768,0.014340443743599786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,512,0.01295377810796102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,1024,32,0.002058666613366869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,256,0.011896000140243106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,128,0.010985777609878115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,16384,0.042945779032177396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,32,0.010758221977286868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,12288,0.03335111008750068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,1024,64,0.01087288889620039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,10240,0.014740443891949125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,16384,0.08112177583906385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,8192,0.013598221871587964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,65536,0.1824444399939643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,12288,0.06363288561503093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,6144,0.011582222249772815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,7168,0.012706666356987424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,10240,0.05470755696296692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,8192,0.046128001477983266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,4096,0.010497777826256223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,7168,0.041760888364579946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,65536,0.3466951052347819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,6144,0.03746133380466037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,3584,0.009319999979601966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,5120,0.03323200013902452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,5120,0.012198222180207571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,2560,0.008585777547624376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,3072,0.008873778084913889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,4096,0.028667555914984807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,2048,0.01126577787929111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,1536,0.009047110875447592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,1024,0.006728888799746831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,3072,0.024380443824662104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,3584,0.026587555805842083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,2560,0.022142221530278523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,768,0.00555644432703654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,2048,0.01961599952644772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,1536,0.017504000001483493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,512,0.004283555679851108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,256,0.0031537777847713898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,128,0.0025840000145965153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,1024,0.015105777316623263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,768,0.014155555102560254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,64,0.002089777754412757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,768,32,0.0022675556441148124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,512,0.013014222184816996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,256,0.011716444459226398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,128,0.011057777537239922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,64,0.010835555692513784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,16384,0.027258666025267705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,768,32,0.010656889114114972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,12288,0.013463999662134381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,65536,0.10649333397547404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,10240,0.011923555698659686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,8192,0.010392888552612728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,7168,0.009588444398509132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,12288,0.06368800004323323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,10240,0.05473155445522732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,6144,0.008740444150235917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,5120,0.00795377790927887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,16384,0.08060710959964328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,7168,0.04176888863245646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,4096,0.00703288863102595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,8192,0.046020444896486074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,65536,0.3239813380771213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,6144,0.037575112448798284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,5120,0.03315555718210008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,3072,0.0063279999627007385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,3584,0.00675733346078131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,2560,0.006022222340106964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,2048,0.005499555418888728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,4096,0.028584000137117174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,3584,0.02652888827853733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,1536,0.008880889250172509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,1024,0.006513777706358168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,768,0.00535999983549118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,3072,0.024294222394625347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,2560,0.022072000635994807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,2048,0.01959111127588484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,512,0.004148444367779626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,1536,0.017458667357762653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,256,0.003007111036115222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,1024,0.015172445111804538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,128,0.002387555523051156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,64,0.002001777705219057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,768,0.014073777529928418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,512,0.012977778083748288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,256,0.011730666789743634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,512,32,0.0022195554855797025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,128,0.011033777561452655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,16384,0.010498666928874122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,65536,0.05239466826121012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,64,0.010932444698280759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,512,32,0.010675555302037133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,12288,0.008952889177534316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,10240,0.008027555214034187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,8192,0.007280888656775157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,7168,0.010238221950001186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,16384,0.08074133263693915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,12288,0.06369600031110975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,10240,0.05471822288301256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,6144,0.0064702220261096954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,8192,0.04601688848601448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,5120,0.008558221989207799
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,4096,0.005505777895450592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,3584,0.00563466673096021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,7168,0.04177422324816386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,6144,0.037596444288889565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,65536,0.317720890045166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,3072,0.006950221955776215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,5120,0.03321155574586656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,2560,0.0064222220745351576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,2048,0.005679111099905438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,4096,0.02864711152182685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,1536,0.0047644442982143825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,3584,0.02664533257484436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,3072,0.024359999431504145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,1024,0.005774222314357758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,2560,0.02199555602338579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,768,0.004863111095296012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,2048,0.019672888848516677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,512,0.003859555555714501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,1536,0.017370666066805523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,1024,0.015228443675571017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,256,0.002887111157178879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,128,0.002373333399494489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,768,0.014124444789356656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,64,0.0021786666992637846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,256,32,0.0021484444538752237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,256,0.011674666570292579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,65536,0.03286044465170966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,128,0.010943111446168689
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,512,0.012889777620633444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,64,0.01086400035354826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,16384,0.010925333533022137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,256,32,0.010659555594126383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,12288,0.00923822240697013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,10240,0.008481777376598781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,8192,0.007487110793590546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,7168,0.007290666302045186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,16384,0.08076622088750203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,10240,0.054824888706207275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,6144,0.006769777586062749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,12288,0.06349422534306844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,8192,0.04578311244646708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,5120,0.0063786668082078295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,7168,0.041745778587129384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,4096,0.005706666658322017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,3584,0.005596444424655702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,3072,0.005172444300519096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,65536,0.3027253415849474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,6144,0.03757244348526001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,5120,0.033124443557527326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,4096,0.028537776735093858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,3584,0.02647022240691715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,2048,0.004549333204825719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,1536,0.004122666600677702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,1024,0.005592000153329637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,3072,0.024396444360415142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,768,0.004646222210592694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,2560,0.022031999296612207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,2048,0.019719999697473314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,1536,0.017331555485725403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,1024,0.015185778339703878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,512,0.0036675557494163513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,2560,0.005000888887378904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,256,0.002781333401799202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,128,0.0022328889204396144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,768,0.01408622165520986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,64,0.002151111140847206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,128,32,0.0021182222084866627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,512,0.012855110896958245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,256,0.01165600038237042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,65536,0.017633777525689866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,128,0.011049778097205691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,16384,0.007878222399287755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,64,0.010896888871987661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,128,32,0.010664888554149203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,12288,0.0069875551594628235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,10240,0.006674666785531574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,8192,0.006045333213276333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,7168,0.005699555493063397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,16384,0.0804248915778266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,12288,0.06334044535954793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,10240,0.05454755491680569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,8192,0.04571377899911669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,6144,0.005223999834722943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,5120,0.005117333183685939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,65536,0.29003556569417316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,4096,0.004604444321658877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,7168,0.04159733321931627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,6144,0.0372631119357215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,3584,0.004870222260554631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,3072,0.004454222404294544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,5120,0.032821334070629544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,2560,0.0047208890318870544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,4096,0.028499556912316218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,3584,0.026260443859630164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,2048,0.004007110993067424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,1536,0.0038524443904558816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,3072,0.024147555232048035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,1024,0.003528000166018804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,2560,0.02186044388347202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,2048,0.01937244501378801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,768,0.004523555437723796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,512,0.003579555700222651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,1536,0.017103999853134155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,256,0.002667555585503578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,1024,0.015099555253982544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,128,0.002238222294383579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,768,0.013850666582584381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,64,0.0021537778278191886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,512,0.012715555727481842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,64,32,0.0021066665649414062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,256,0.011531555818186866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,128,0.010928000013033548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,65536,0.016265778077973258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,64,0.010632889138327705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,12288,0.0058906666106647914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,64,32,0.010799110763602786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,16384,0.0069368887278768755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,10240,0.005975111077229182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,8192,0.005434666656785541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,7168,0.00488800017370118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,12288,0.0625422199567159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,6144,0.0047662220895290375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,16384,0.07955555783377753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,10240,0.05381066600481669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,5120,0.004825777891609403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,7168,0.04122489028506809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,4096,0.0042453333735466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,8192,0.04511022236612108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,3584,0.004290666845109728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,6144,0.036806222465303205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,65536,0.29339999622768825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,5120,0.032587554719712995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,2560,0.00444088876247406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,2048,0.0037235555549462638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,4096,0.02819733487235175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,3584,0.026081777281231348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,3072,0.0041475556790828705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,1536,0.003673777812057071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,3072,0.02392622166209751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,1024,0.0034222222036785553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,2560,0.02162666618824005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,768,0.003548444559176763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,512,0.003569777641031477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,2048,0.0192284451590644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,256,0.002854222224818336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,1536,0.017045333981513977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,128,0.00222044438123703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,1024,0.014779556128713818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,768,0.013917333549923368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,64,0.0021262222694026097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,512,0.012656889028019376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,256,0.011507555842399597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,128,32,32,0.002104000084929996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,128,0.010849778023031024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,64,0.010678222609890832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,128,32,32,0.010816000401973724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,10240,1.2919901741875541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,12288,1.5695066452026367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,8192,1.6618462668524847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,8192,0.9848871231079102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,16384,2.0535492367214627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,10240,2.0500409868028426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,7168,0.8180533515082465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,12288,2.7127715216742625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,5120,0.5687822235955132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,6144,0.7273955345153809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,7168,1.4468257692125108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,6144,1.284790250990126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,5120,1.0460116068522136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,16384,3.2555014292399087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,3584,0.4032879935370551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,4096,0.48694223827785915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,4096,0.8457493252224393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,3072,0.6493608686659071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,2560,0.5411181979709202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,2560,0.327639102935791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,2048,0.4389653205871582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,3584,0.8186640209621854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,1536,0.3369564480251736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,2048,0.2715902328491211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,3072,0.38175733884175617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,768,0.19155732790629068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,256,0.05240622162818909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,512,0.11730133162604438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,512,0.10575822326872085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,1536,0.21984355979495576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,1024,0.15909600257873535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,64,0.019155555301242404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,1024,0.2511928876241048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,256,0.09903022315767075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,128,0.026751110951105755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,128,0.10108444425794814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,65536,32,0.01112711098459032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,768,0.12889688544803196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,64,0.09767733017603557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,65536,32,0.09963466723759969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,12288,0.4111288918389215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,16384,0.5678008927239312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,10240,0.5950729052225748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,12288,0.7863119973076714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,10240,0.3458426793416341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,16384,0.942479133605957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,7168,0.2476968765258789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,8192,0.30970311164855957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,8192,0.4885706901550293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,6144,0.3272791173723009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,7168,0.38516444630093044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,4096,0.2166293329662747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,5120,0.2948755423227946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,5120,0.18959111637539336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,6144,0.222671111424764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,65536,1.8152781592475042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,3584,0.1893528832329644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,4096,0.1419866614871555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,3072,0.1618666648864746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,2560,0.13088977336883545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,3072,0.09846044249004787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,3584,0.12289067109425862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,2560,0.09155555566151936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,1024,0.05462133222156101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,2048,0.07622577746709187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,1536,0.08900355630450779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,512,0.017792888813548617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,768,0.046464890241622925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,1024,0.047445333666271634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,2048,0.11704444885253906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,768,0.027959111664030287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,256,0.007495111061467065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,1536,0.06174311372968885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,128,0.005579555614127054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,64,0.004571555389298333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,512,0.0247688889503479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,256,0.022104889154434204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,32,0.003912888881233004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,128,0.021057777934604224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,64,0.019976889093716938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,16384,32,0.01965777741538154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,16384,65536,3.2773253122965493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,12288,0.32010666529337567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,10240,0.3980720043182373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,16384,0.44213154580858016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,16384,0.6432764265272353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,12288,0.5289262135823568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,10240,0.2658284505208333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,8192,0.20979555447896323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,7168,0.1818551090028551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,8192,0.32670667436387807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,6144,0.2587795522477892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,6144,0.1512640052371555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,5120,0.21298578050401476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,7168,0.31850043932596844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,4096,0.16978932751549614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,65536,1.4366221957736547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,3584,0.1587031152513292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,5120,0.12970844904581705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,3584,0.08658311102125381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,3072,0.07659822040134005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,4096,0.10618578063117133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,3072,0.12220799922943115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,1024,0.042690667841169566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,2048,0.0905111100938585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,2048,0.05588889122009277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,2560,0.0738311078813341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,768,0.032687998480266996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,1536,0.04767733481195238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,2560,0.11243911584218343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,1024,0.0258577780591117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,256,0.007032000356250339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,768,0.023594665858480666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,1536,0.06900533040364583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,64,0.0040737779604064096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,512,0.01075111081202825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,512,0.022580444812774658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,32,0.0033048888047536216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,256,0.019742222295867074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,64,0.018201778332392376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,128,0.005158222383923001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,128,0.019300444258583915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,12288,32,0.01787644459141625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,12288,65536,2.4702347649468317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,12288,0.2663315667046441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,16384,0.3474871052636041
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,12288,0.4341751204596625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,16384,0.5731582111782497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,10240,0.3431804445054796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,10240,0.21776178148057726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,8192,0.27828889422946507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,8192,0.16940089066823324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,6144,0.20868266953362358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,65536,1.2056631512112088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,7168,0.2594960000779894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,6144,0.12864443990919325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,5120,0.09970311323801677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,7168,0.15246844291687012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,4096,0.1320755614174737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,5120,0.185917337735494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,4096,0.08234755860434638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,3584,0.11533066961500381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,3072,0.11164711581336127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,2560,0.09363200267155965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,2048,0.0685031082895067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,2560,0.057151112291547984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,3584,0.08056888977686565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,1536,0.05775555637147692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,1024,0.035375111632876925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,768,0.014144889182514615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,3072,0.0724417765935262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,2048,0.05246666736072964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,1024,0.02498311135503981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,768,0.023081777824295893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,1536,0.035255110926098294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,512,0.010667555862002902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,128,0.005103111267089844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,512,0.02160266704029507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,256,0.00795288880666097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,256,0.01901333365175459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,64,0.0038497779104444715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,128,0.018024888303544786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,64,0.01756088932355245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,65536,2.1571085188123917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,10240,32,0.01759999990463257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,10240,32,0.003444444388151169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,12288,0.3488604492611355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,16384,0.2771208816104465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,16384,0.4179671075608995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,12288,0.2288435565100776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,10240,0.2904000017378065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,7168,0.18662310971154106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,8192,0.12230488989088271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,8192,0.2181537813610501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,10240,0.1773644420835707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,7168,0.11571021874745686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,5120,0.1306497785780165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,6144,0.09174755546781753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,6144,0.16374044948154026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,65536,0.9836675855848523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,3584,0.09032889207204182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,4096,0.11105689075258042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,3072,0.07851999998092651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,4096,0.07152622275882296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,5120,0.0859911110666063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,1536,0.045532445112864174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,2560,0.06569955746332805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,3584,0.06363466713163587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,3072,0.05678755707210965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,2560,0.045287112394968666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,1024,0.015503999259736804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,2048,0.03822844558291965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,512,0.007912889122962952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,256,0.005202666752868229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,65536,1.625935130649143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,1536,0.02197244432237413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,768,0.01016533292002148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,1024,0.019189332922299702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,2048,0.05958933300442166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,768,0.018030222919252183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,128,0.0041244443919923575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,512,0.016584000653690763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,64,0.0033813334173626373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,8192,32,0.003077333379122946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,64,0.013936888840463428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,256,0.01514400045077006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,32,0.013939556148317126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,8192,128,0.014280888769361707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,16384,0.24606222576565215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,12288,0.17812621593475342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,10240,0.234132448832194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,12288,0.3075875441233317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,16384,0.37553776635064023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,8192,0.19176622231801352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,7168,0.16266666518317327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,10240,0.14562133948008218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,7168,0.09256177478366429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,6144,0.15370221932729086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,5120,0.11348266071743435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,4096,0.09163644578721787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,65536,0.876348442501492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,5120,0.07601244582070245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,8192,0.11534844504462348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,4096,0.058746669027540416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,6144,0.0811777777141995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,2560,0.05786044730080498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,3072,0.04639377858903673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,3072,0.076873779296875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,2048,0.04748622245258755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,2560,0.040064887868033514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,3584,0.08894844187630548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,3584,0.05662044551637438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,1536,0.040080888403786555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,2048,0.024124445186720953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,65536,1.4572551515367296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,1024,0.011008888483047485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,768,0.009918221996890174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,512,0.007343110938866933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,1024,0.01851288808716668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,256,0.00509066672788726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,128,0.004458666675620609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,768,0.017461333009931777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,1536,0.02068800065252516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,64,0.0033520000676314035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,256,0.014783999986118741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,7168,32,0.0030053332448005676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,64,0.013243555194801755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,128,0.014000889327791003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,32,0.013655111193656921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,7168,512,0.015980444020695157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,16384,0.2030657794740465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,12288,0.14033333460489908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,12288,0.2703324423895942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,16384,0.3602195580800374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,10240,0.22320355309380424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,10240,0.10956799983978271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,8192,0.1610835525724623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,8192,0.08921155664655898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,7168,0.15463199880388048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,6144,0.12068533897399902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,65536,0.7593875461154513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,5120,0.09824177953932021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,6144,0.06952533457014295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,7168,0.0859946674770779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,3584,0.06833244694603814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,4096,0.05041244294908312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,5120,0.06528266933229235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,2560,0.04987288845909966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,4096,0.07917244566811456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,3584,0.04534399840566847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,65536,1.2611573537190754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,3072,0.0665902230474684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,1536,0.03143999973932902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,1024,0.01037688884470198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,2048,0.021586666504542034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,768,0.008839999636014303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,3072,0.04062844316164652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,1024,0.017782222893502977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,2560,0.030976000759336684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,512,0.006866666591829724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,256,0.004922666483455234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,768,0.016456888781653512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,128,0.0038897775941424896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,512,0.015467555986510383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,256,0.014156444205178155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,64,0.0032151111712058387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,1536,0.01994044416480594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,2048,0.04550310969352722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,128,0.013237333132161034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,6144,32,0.0028088889602157804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,64,0.01293155550956726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,6144,32,0.012973333398501078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,16384,0.16903466648525664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,12288,0.222088893254598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,10240,0.17661689387427437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,16384,0.30494756168789333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,12288,0.12327910794152154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,7168,0.11851289537217881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,8192,0.07993421951929729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,10240,0.10342844327290852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,8192,0.13868355751037598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,7168,0.0752684473991394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,6144,0.1152062283621894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,5120,0.08739378054936726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,6144,0.061962664127349854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,4096,0.07053333520889282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,3584,0.0633173320028517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,5120,0.058339556058247886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,65536,0.6707368956671821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,3072,0.029126223590638902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,3072,0.05011111166742113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,65536,1.0572675069173176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,2560,0.04642044504483541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,1536,0.013609777722093793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,4096,0.04520533482233683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,2048,0.03399466805987888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,2560,0.023558222585254248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,768,0.00850666645500395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,1024,0.010199999643696679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,3584,0.03957866628964742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,512,0.006600889066855113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,1536,0.01924622224436866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,1024,0.017176000608338248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,2048,0.02144177754720052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,128,0.0037493333220481873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,512,0.014967999524540372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,64,0.003026666740576426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,256,0.01370933320787218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,256,0.005585777676767773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,5120,32,0.0026746667507621977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,128,0.01293333371480306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,768,0.016134222348531086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,32,0.012703999876976013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,5120,64,0.012656889028019376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,16384,0.22324887911478677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,16384,0.12755289342668322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,12288,0.09692444403966267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,12288,0.17586844497256807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,8192,0.10574844148423936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,10240,0.08174400197135077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,10240,0.14928444226582846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,7168,0.09298488828870986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,8192,0.06831022103627522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,6144,0.0810026658905877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,65536,0.5562017758687338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,5120,0.06807733244366117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,7168,0.060157332155439586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,5120,0.04765511221355862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,65536,0.8579911126030816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,4096,0.035663998789257474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,3584,0.027207111318906147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,6144,0.057592001226213246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,4096,0.05599288807974922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,3072,0.0381991101635827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,3072,0.02490933405028449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,2048,0.013678222066826291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,2048,0.0203795548942354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,1536,0.011003555523024665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,3584,0.049485332436031766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,2560,0.022786666949590046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,768,0.006813333266311222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,1024,0.008075555165608725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,512,0.007052444749408298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,256,0.004951999833186467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,1024,0.016241777274343703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,768,0.01531733406914605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,2560,0.03626311156484816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,64,0.0027831111931138565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,256,0.012999110751681857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,1536,0.018457776970333524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,128,0.0033697777738173804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,4096,32,0.002461333241727617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,64,0.011735111474990845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,128,0.011885333392355176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,32,0.011885333392355176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,4096,512,0.014176888598336114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,12288,0.14892889393700495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,16384,0.19744533962673613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,16384,0.11512000030941433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,12288,0.08816533618503147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,10240,0.11804978052775066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,8192,0.09549955526987712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,7168,0.08386400010850693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,10240,0.07521600193447538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,8192,0.062104887432522245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,7168,0.05552444524235196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,6144,0.07292266686757405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,65536,0.5126541985405816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,5120,0.060236447387271456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,6144,0.049512889650132924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,4096,0.04961688982115852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,5120,0.04235644472969902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,3072,0.03372800019052293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,3584,0.04393600092993843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,3584,0.026787555880016748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,2048,0.012146666646003723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,2560,0.024125332633654278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,65536,0.7561226420932345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,1536,0.010241777532630498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,1024,0.007622222105662028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,3072,0.024559110403060913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,768,0.006523555351628198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,2560,0.022287999590237934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,1536,0.01824977828396691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,1024,0.01588800052801768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,512,0.005478222337034013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,256,0.004655999855862724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,4096,0.028744889630211726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,64,0.0027644443843099806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,2048,0.019955555597941082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,768,0.014885332849290637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,512,0.013930666777822705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,128,0.011773332953453064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,256,0.012734221915404001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,32,0.0026133333643277488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3584,128,0.003212444484233856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,64,0.011415111521879831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3584,32,0.011372444530328115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,12288,0.12131555875142415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,10240,0.10181866751776801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,16384,0.10467644532521565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,16384,0.16861155298021105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,12288,0.0869235528839959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,10240,0.06864533159467909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,8192,0.08304799927605523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,7168,0.07259733147091336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,8192,0.05737421909968058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,5120,0.054133332437939115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,7168,0.05084177851676941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,6144,0.04560888806978861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,65536,0.46685420142279727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,6144,0.06326933039559259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,4096,0.04295377930005392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,5120,0.03346133232116699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,3584,0.03815999958250258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,65536,0.6541973219977485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,3072,0.0336151123046875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,4096,0.02840977907180786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,2560,0.01371022231049008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,2048,0.01109155515829722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,1536,0.0092275556590822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,1024,0.0069360000391801195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,2048,0.019706666469573975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,3584,0.026532444689008925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,1536,0.017863111363516915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,1024,0.015509333875444202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,512,0.00489066665371259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,3072,0.024280000064108107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,256,0.004224888980388641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,2560,0.021935999393463135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,64,0.00250133333934678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,128,0.0030737777964936364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,768,0.005954666684071223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,512,0.013538666897349887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,768,0.014602666099866232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,256,0.012416000167528788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,3072,32,0.0023431111541059283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,128,0.011391999820868174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,64,0.011074666347768573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,3072,32,0.011056000159846412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,12288,0.09903822342554729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,16384,0.13187732961442736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,16384,0.09573066896862453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,8192,0.06789688931571113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,12288,0.07190755340788099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,10240,0.06340889135996501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,10240,0.08398577902052139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,8192,0.0522346662150489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,7168,0.06044355365965101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,6144,0.05214044451713562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,65536,0.534007125430637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,5120,0.044574220975240074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,4096,0.03680800067053901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,65536,0.442122671339247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,7168,0.045279999574025474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,3584,0.028472890456517536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,6144,0.03663555449909634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,3072,0.023100444012218054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,3584,0.02611733310752445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,2560,0.01941333380010393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,3072,0.0240346673462126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,1536,0.012599111431174807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,4096,0.028488000233968098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,2560,0.02179199953873952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,5120,0.03243911266326904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,2048,0.01959466603067186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,1024,0.00904977818330129
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,768,0.0074311114019817775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,512,0.005713777823580637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,2048,0.015013333823945789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,256,0.004209777961174647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,1536,0.017542221479945712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,768,0.014476444986131458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,64,0.0025066667132907445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,1024,0.015619556109110514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,128,0.0031911111954185697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,512,0.01348622226052814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2560,32,0.0022604444788561925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,256,0.012141333685980903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,128,0.011223110887739392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,64,0.010943999720944298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2560,32,0.011092444260915121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,16384,0.10470577743318345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,16384,0.08713422218958537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,12288,0.08033688863118489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,10240,0.0673360029856364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,12288,0.06499377886454265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,8192,0.05503733290566338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,7168,0.04874044325616625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,6144,0.04243200023969015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,10240,0.05738933218850029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,65536,0.4325608942243788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,7168,0.04050666756100125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,5120,0.0362408889664544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,6144,0.03644533289803399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,8192,0.04563644528388977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,65536,0.4161262247297499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,4096,0.01590044465329912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,3584,0.012395555774370829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,3072,0.021631111701329548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,4096,0.028016000986099243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,3584,0.02574577762020959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,2048,0.014527999692493014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,3072,0.024010666542583044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,2560,0.02141511109140184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,5120,0.03220444586541917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,1536,0.01183733344078064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,768,0.007274666594134436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,2048,0.019357333580652874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,512,0.00552977787123786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,1536,0.017367111312018502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,768,0.013997332917319404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,1024,0.015088889333936902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,1024,0.008618666893906063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,256,0.004091555459631814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,128,0.002983999955985281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,64,0.0023893333143658107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,512,0.013076444466908773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,32,0.0021466666625605687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,256,0.011886222494973077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,128,0.010949333508809408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,2048,2560,0.018606222338146634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,64,0.01076533314254549
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,2048,32,0.010843555960390302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,16384,0.08076533344056872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,10240,0.05232622226079305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,12288,0.06191466914282905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,8192,0.04283555679851108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,12288,0.06159289015663994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,16384,0.07959644662009345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,7168,0.037816001309288874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,6144,0.031042665243148804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,10240,0.052785777383380465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,65536,0.33258401023017037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,8192,0.044658667511410184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,5120,0.01277599980433782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,4096,0.01108000013563368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,7168,0.040456000301573015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,65536,0.37615466117858887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,3584,0.010584000084135266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,6144,0.03646399908595615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,3072,0.010031110710567897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,2560,0.017130666308932833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,3584,0.02586666742960612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,2048,0.014141332772043018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,3072,0.02367555598417918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,4096,0.02805244591501024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,1536,0.011179555621412067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,5120,0.03209422363175286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,1024,0.008186666501892937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,2560,0.021516443954573736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,512,0.005271111097600725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,1536,0.017303999927308824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,768,0.006798222247097228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,1024,0.015224888920783997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,256,0.003818666769398583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,768,0.014141332772043018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,2048,0.01920977731545766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,128,0.0029031110720502008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,64,0.002313777804374695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1536,32,0.0020906666500700847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,512,0.013086222112178802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,128,0.011071110765139261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,256,0.011778666741318174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,64,0.010818666881985135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1536,32,0.010688888529936472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,16384,0.05466577741834852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,12288,0.04232355621125963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,10240,0.03605422377586365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,65536,0.23468266593085396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,16384,0.07795644468731351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,8192,0.020548444655206468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,7168,0.01400888959566752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,12288,0.06099200248718262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,10240,0.052914665804968945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,6144,0.012591111163298288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,8192,0.04460444384151035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,5120,0.01017777787314521
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,65536,0.32961511611938477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,7168,0.04053422146373325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,3584,0.018375999397701688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,3072,0.016168889072206285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,5120,0.032241778241263494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,4096,0.027975112199783325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,6144,0.036299556493759155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,2560,0.013761777844693927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,2048,0.01131377783086565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,4096,0.009093333449628618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,1536,0.009092444347010719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,1024,0.006664888726340399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,3584,0.02570666703912947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,2560,0.021487111846605938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,3072,0.023740443918440077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,2048,0.01921422282854716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,1536,0.01724177764521705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,768,0.005541333307822545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,512,0.004317333300908406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,256,0.003156444471743372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,128,0.0024968888610601425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,64,0.0021537778278191886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,1024,0.01517511076397366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,768,0.013946667313575745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,1024,32,0.0018702221827374564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,256,0.01178311142656538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,512,0.01293511109219657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,128,0.011062222222487131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,64,0.010753778119881948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,1024,32,0.010675555302037133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,16384,0.042825778325398765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,12288,0.03327644533581204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,10240,0.012870222330093384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,65536,0.17490222718980578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,12288,0.060886224110921226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,8192,0.012423999607563019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,16384,0.0777075555589464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,10240,0.052770667605929904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,8192,0.044512000348832875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,6144,0.01069155583779017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,5120,0.014977777997652689
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,7168,0.011615110768212212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,65536,0.3038693269093831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,4096,0.01221866657336553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,7168,0.0404151115152571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,3584,0.011582222249772815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,6144,0.03623288869857788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,3072,0.00832711077398724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,5120,0.0321084459622701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,2560,0.008048888709810045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,1536,0.008933333059151968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,4096,0.027832888894610937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,2048,0.011281777587201862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,3584,0.02574311031235589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,3072,0.023583110835817125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,2560,0.021342222889264423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,1024,0.006613333192136552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,2048,0.019188443819681805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,256,0.003072000005178981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,768,0.005444444302055571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,512,0.0042071110672420925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,1536,0.01705777810679542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,1024,0.015028445257080926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,768,0.013976888524161445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,128,0.002437333265940348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,64,0.0020497778637541667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,512,0.012820444173283048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,768,32,0.002218666589922375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,256,0.011629333098729452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,16384,0.017352888981501263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,128,0.011063999599880643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,64,0.010807111031479306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,768,32,0.010610666539933948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,12288,0.020498666498396132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,65536,0.10550577772988214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,10240,0.017308443784713745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,8192,0.014464888307783337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,16384,0.0776488913430108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,7168,0.013947554760509066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,12288,0.060905781057145864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,10240,0.052689777480231396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,6144,0.01237422227859497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,5120,0.011023999916182624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,7168,0.040439999765819974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,4096,0.009377777576446533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,65536,0.28583378261990017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,6144,0.036306666003333196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,3584,0.009144888983832465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,5120,0.03205777870284186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,3072,0.008407110969225565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,8192,0.04433511031998528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,2560,0.007610666255156199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,4096,0.02780533168050978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,3584,0.025776889589097764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,2048,0.0068853331936730286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,1536,0.008755555583371056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,1024,0.0064328888224230866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,3072,0.023733332753181458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,768,0.0052666668262746595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,2560,0.021353777911927965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,2048,0.01902222302224901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,512,0.004107555581463708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,1536,0.017120889491505094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,256,0.00300444434914324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,128,0.00239644447962443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,1024,0.014979556202888489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,64,0.0019911111642917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,512,0.012776888906955719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,512,32,0.0022071111533376905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,768,0.013979555832015144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,256,0.011591999895042844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,128,0.011034666664070554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,65536,0.05279111199908786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,16384,0.014319111903508505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,64,0.010830221904648675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,12288,0.012078222301271228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,512,32,0.010650667051474253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,10240,0.010896888871987661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,8192,0.009219555391205682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,16384,0.07734222544564141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,7168,0.009017777939637503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,12288,0.060862223307291664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,10240,0.052816000249650746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,6144,0.008107555409272512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,8192,0.04439466529422336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,7168,0.04043111205101013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,5120,0.007736889024575551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,4096,0.006796444455782573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,65536,0.2780373361375597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,3584,0.007009777757856581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,6144,0.03618933426009284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,5120,0.03186666634347703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,3072,0.006437333507670297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,2560,0.0058355554938316345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,4096,0.027777777777777776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,3584,0.02570044497648875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,2048,0.005625777774386936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,3072,0.023716444770495098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,1536,0.004619555754794015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,1024,0.00572622236278322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,2560,0.021232888102531433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,768,0.004763555609517627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,2048,0.019009777241282992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,512,0.003742222156789568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,256,0.0028320000403457214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,1536,0.017083555459976196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,1024,0.014936000108718872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,128,0.00231555559568935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,64,0.0021822222818930945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,768,0.01407199932469262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,512,0.012699555191728802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,256,32,0.0021359999146726397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,256,0.011681777735551199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,16384,0.009582222335868413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,128,0.011035555766688453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,65536,0.026709333062171936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,64,0.01073155552148819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,256,32,0.010651555326249866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,12288,0.008400888906584846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,10240,0.007779555188284979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,8192,0.0070257774657673305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,16384,0.07724622223112318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,7168,0.007083555890454187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,6144,0.006500444478458828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,12288,0.06094933218426175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,10240,0.052807112534840904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,8192,0.04435555471314324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,5120,0.005961777849329843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,4096,0.005396444350481033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,7168,0.04034488730960422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,65536,0.2776693238152398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,3584,0.0052888890107472735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,6144,0.03625955515437656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,3072,0.0052044445441828836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,5120,0.032067557175954185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,2560,0.004770666774776247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,2048,0.004649777793222004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,4096,0.027817779117160376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,3584,0.02574577762020959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,3072,0.023476445012622412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,1024,0.005526222288608551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,1536,0.004059555629889171
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,2560,0.021363554729355708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,2048,0.019127110640207928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,768,0.004572444491916233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,512,0.003629333443111844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,1536,0.017055999901559617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,256,0.002732444347606765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,1024,0.014923555983437432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,128,0.0022417778770128884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,768,0.01402311192618476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,32,0.0021155555215146807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,128,64,0.002161777681774563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,512,0.012829333543777466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,256,0.011518222590287527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,128,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,16384,0.00741155528359943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,65536,0.014726221561431885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,64,0.010699555277824402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,12288,0.0069066666894488865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,128,32,0.010830221904648675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,10240,0.006088888893524806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,8192,0.006193777753247156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,7168,0.00554577757914861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,16384,0.07706577910317315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,6144,0.005215999980767568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,12288,0.06066044171651205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,10240,0.052405334181255765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,8192,0.04411555661095513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,5120,0.0047439999050564235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,7168,0.0400622222158644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,4096,0.004899555610285865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,65536,0.276439110438029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,3584,0.004688888788223267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,3072,0.004368888835112254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,6144,0.03581155671013726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,2560,0.004138666722509596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,5120,0.03172977765401205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,4096,0.027464888162083093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,2048,0.0038746665749284956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,1536,0.003793777690993415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,3584,0.025477333201302424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,3072,0.023396443989541795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,1024,0.0035111109415690103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,2048,0.018984888990720112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,768,0.004516444272465176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,1536,0.016873778568373788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,2560,0.021055999729368422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,512,0.003584889074166616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,1024,0.01481066644191742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,256,0.00281866660548581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,128,0.002234666711754269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,768,0.013562666873137156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,64,0.0021217777911159727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,64,32,0.0021075554605987337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,512,0.012661333713266583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,256,0.011419555379284753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,65536,0.016182222300105624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,128,0.01090844472249349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,16384,0.006849777781301075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,64,0.010869333313571082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,12288,0.005746666755941179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,64,32,0.010661332971519895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,10240,0.0059057776298787855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,8192,0.0053680001033677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,7168,0.005137777576843898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,6144,0.004657777647177379
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,10240,0.05270933442645603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,12288,0.06095555755827162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,16384,0.0778240031666226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,8192,0.044215109613206655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,5120,0.004423111263248655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,4096,0.0041546668443414895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,65536,0.2759653462304009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,7168,0.04014222158326043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,3584,0.004846222284767362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,3072,0.0041164445380369825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,6144,0.03602133194605509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,2560,0.004408888932731417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,5120,0.03185866607560052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,4096,0.02742044462098016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,2048,0.003702222059170405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,3584,0.025302223033375208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,3072,0.023165333602163527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,1536,0.003612444632583194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,1024,0.00342399999499321
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,768,0.0035048888789282907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,512,0.003943110919660992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,2560,0.020961778031455144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,2048,0.01881866653760274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,1536,0.016788444585270353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,256,0.002814222127199173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,1024,0.014528888795110913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,128,0.0022275555464956495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,64,0.0021315556433465746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,768,0.013734222286277346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,96,32,32,0.002104000084929996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,512,0.012472000386979846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,256,0.01143022212717268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,128,0.010847110715177325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,64,0.01052177780204349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,96,32,32,0.01056977775361803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,10240,1.2801627053154838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,12288,1.5634302563137477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,8192,1.6575146781073675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,16384,2.0459910498725042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,10240,2.044815911187066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,8192,0.9753671222262912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,7168,0.8090186648898654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,12288,2.706146664089627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,6144,0.6811217731899686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,5120,0.6019075711568197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,6144,1.277774175008138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,7168,1.5291510687934027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,5120,1.0413831075032551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,16384,3.2457076178656687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,4096,0.4590915573967828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,4096,0.8415164417690701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,3584,0.40312711397806805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,3584,0.7384826872083875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,3072,0.3542204433017307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,2560,0.30513244205051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,2048,0.43427289856804746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,2560,0.5363804499308268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,1024,0.22995111677381727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,1536,0.33253688282436794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,3072,0.7083199818929037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,512,0.1047475536664327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,2048,0.2669724358452691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,256,0.05247999893294441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,128,0.02663822140958574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,1024,0.15709866417778864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,64,0.011316444310877057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,768,0.1919484403398302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,1536,0.21652444203694662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,512,0.10280888610415989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,65536,32,0.01147377739350001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,768,0.12695111168755427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,128,0.09874933295779759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,64,0.09678755866156684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,32,0.09398577610651652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,65536,256,0.0983733336130778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,12288,0.43582844734191895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,16384,0.5579137802124023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,10240,0.5927333301968044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,12288,0.7822462187872993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,16384,0.9402959611680773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,10240,0.3699324395921495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,8192,0.48839733335706925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,8192,0.30405065748426646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,7168,0.24488443798489043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,7168,0.36769866943359375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,6144,0.360319111082289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,65536,1.7936710781521266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,4096,0.21535821755727133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,5120,0.29273422559102374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,6144,0.22945332527160645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,5120,0.18644089168972441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,3584,0.11268533600701226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,3584,0.18878844049241808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,3072,0.159661332766215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,2048,0.10546933280097114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,2560,0.14626222186618382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,4096,0.14370400375790066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,3072,0.10661866929796006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,2048,0.07106577687793307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,1024,0.06019022067387899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,2560,0.09108089076148139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,1536,0.057256890667809375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,512,0.015969778100649517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,1024,0.04654133319854736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,768,0.04655111167165968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,1536,0.08906400203704834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,256,0.00741511086622874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,768,0.02903999884923299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,512,0.024047111471494038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,256,0.021951110826598272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,64,0.004548444516128964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,128,0.02074044446150462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,128,0.005571555760171678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,32,0.0037546666959921518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,32,0.0199937770764033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,16384,64,0.020032000210550096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,12288,0.31625244352552623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,16384,65536,3.263700485229492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,16384,0.6394809087117513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,16384,0.4336640040079753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,12288,0.5262124273512099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,10240,0.420510212580363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,10240,0.26003021664089626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,8192,0.3219288984934489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,7168,0.2838311195373535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,7168,0.1815119981765747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,6144,0.1462684472401937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,6144,0.25398845142788357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,8192,0.22574933369954428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,4096,0.1693697770436605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,3584,0.14052355289459229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,3584,0.08644977543089126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,4096,0.09536888864305283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,5120,0.1290373272365994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,5120,0.22979733679029676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,3072,0.07677955759896173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,2560,0.1026337809032864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,65536,1.4506248897976344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,2560,0.06569244464238484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,2048,0.08247822523117065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,3072,0.13620089160071477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,768,0.03269066744380527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,2048,0.06146933635075887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,1024,0.025603555970721777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,512,0.010630221830474006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,1536,0.04658755660057068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,256,0.006934222247865465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,1536,0.06894844770431519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,128,0.005104889058404499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,768,0.024324445260895625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,512,0.02243288854757945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,1024,0.04730133215586344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,32,0.0031395554542541504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,64,0.004030222280157937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,128,0.01865688959757487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,64,0.018198221921920776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,256,0.019506666395399306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,12288,32,0.01924711134698656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,12288,65536,2.4609502156575522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,16384,0.340939548280504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,12288,0.2623857657114665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,12288,0.43193867471483016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,16384,0.5389946831597222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,10240,0.21395822366078696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,10240,0.37364355723063153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,8192,0.2688648965623644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,7168,0.23508532842000326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,8192,0.16739732689327666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,6144,0.2083982229232788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,5120,0.16800178421868217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,6144,0.1286311149597168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,7168,0.14986488554212782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,4096,0.13197333282894558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,5120,0.09887733062108357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,3584,0.11495733261108398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,65536,1.1888266669379342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,3072,0.06582044230567084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,3072,0.09958489073647393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,4096,0.08975111113654242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,2048,0.068031112353007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,2560,0.057987557517157666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,2048,0.04828622274928623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,1536,0.057929780748155385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,2560,0.09387999773025513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,3584,0.0809075567457411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,1024,0.03536266750759549
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,768,0.014177777700954013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,512,0.010562666588359408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,256,0.007696000238259633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,768,0.022847111026446026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,1024,0.024725332856178284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,128,0.005074666606055366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,64,0.0035377778112888336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,512,0.021523555119832356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,256,0.019676445258988273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,32,0.0028924445311228433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,128,0.018020444446139865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,64,0.017811555001470778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,32,0.018530666828155518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,10240,1536,0.03623022304640876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,10240,65536,2.05642212761773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,16384,0.4170488781399197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,10240,0.2623048888312446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,12288,0.20415111382802328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,12288,0.34798044628567165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,16384,0.29525687959459096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,10240,0.16265066464742026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,8192,0.2168053388595581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,7168,0.18592354986402723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,8192,0.12143554952409531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,7168,0.11430844995710586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,6144,0.1643128924899631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,65536,0.9722853766547309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,5120,0.14575289355383977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,5120,0.07778666416803996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,4096,0.11706844965616862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,3072,0.07810311184989081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,3584,0.05787822273042467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,6144,0.09986311197280884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,4096,0.0704524450831943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,3584,0.1012808879216512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,3072,0.051389333274629384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,2560,0.06562399864196777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,1536,0.04556266797913445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,2048,0.05942222144868639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,2048,0.035792890522215105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,768,0.009540444446934594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,1536,0.021386666430367365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,2560,0.04487733377350701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,1024,0.0188391109307607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,768,0.017812444104088675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,256,0.005559999909665849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,1024,0.011828444070286222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,512,0.007558222446176741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,65536,1.6140755547417536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,64,0.003504000190231535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,512,0.016553777787420485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,32,0.0030871110243929755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,128,0.01420088940196567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,256,0.015483554866578845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,32,0.014142221874660917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,8192,64,0.014064000712500678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,8192,128,0.0040693332751592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,16384,0.24139645364549425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,16384,0.36875377760993105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,12288,0.30598754352993435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,10240,0.14482311407725015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,12288,0.1874915493859185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,10240,0.2559448877970378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,8192,0.1902017725838555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,7168,0.1615742180082533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,8192,0.11210489273071289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,7168,0.09208444754282634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,6144,0.08057333363427056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,5120,0.06855466630723742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,5120,0.11454399426778157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,65536,0.861021359761556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,6144,0.1575911045074463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,4096,0.0917466680208842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,4096,0.057641777727339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,3072,0.0689528915617201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,2048,0.04716622167163425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,3584,0.088228444258372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,2560,0.04019644525316026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,2560,0.06444978051715426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,2048,0.02327377763059404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,1536,0.03608444333076477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,1536,0.02014222244421641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,1024,0.010510221951537661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,65536,1.431632889641656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,768,0.009495110975371467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,3584,0.055204444461398654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,512,0.007101332975758447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,768,0.01698933376206292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,1024,0.018186666899257235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,128,0.004285333471165763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,64,0.0032684444967243406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,3072,0.05078666739993625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,512,0.016006223029560514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,32,0.003091555502679613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,256,0.015102222561836243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,128,0.013383111192120446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,64,0.013462222284740873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,7168,32,0.0136817776494556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,7168,256,0.005049777941571342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,16384,0.32210667928059894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,16384,0.19935822486877441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,12288,0.26531823476155597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,12288,0.14978222052256265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,8192,0.16106311480204263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,10240,0.2200942171944512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,8192,0.08852710988786484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,10240,0.11819556024339463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,65536,0.7370710902743869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,5120,0.0993751088778178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,6144,0.13606044981214735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,6144,0.06896177927652995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,7168,0.15385954909854466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,7168,0.0773822201622857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,5120,0.05954133139716255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,4096,0.07891555627187093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,3072,0.059664891825781934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,2560,0.04972622129652235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,3584,0.06837599807315402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,65536,1.253875520494249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,3072,0.04003289010789659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,3584,0.04857244425349765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,2560,0.027228444814682007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,2048,0.021454221672481958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,4096,0.05432977941301134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,1536,0.019554666346973844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,768,0.008567999634477828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,1024,0.010139555566840703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,512,0.006694222076071634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,2048,0.040543999936845564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,1024,0.017272000511487324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,128,0.003823111040724648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,768,0.016413332687483895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,512,0.015278221832381355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,256,0.013983110586802164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,64,0.00295733329322603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,32,0.002784888984428512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,256,0.004794666750563515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,128,0.012970666918489667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,64,0.013040000365840064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,6144,32,0.013175110850069257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,6144,1536,0.032280001375410296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,16384,0.16421333948771158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,16384,0.2766053411695692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,12288,0.11250666777292888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,10240,0.1908444431093004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,12288,0.23985332912868926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,8192,0.1294355524910821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,10240,0.09405333466000027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,8192,0.07810311184989081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,65536,0.6322613292270237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,7168,0.1323431067996555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,5120,0.0869084464179145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,6144,0.11473600069681804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,7168,0.07428889142142402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,6144,0.061703112390306264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,5120,0.05318489008479648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,4096,0.06572622060775757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,3584,0.05663022067811754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,2560,0.04158666729927063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,3072,0.04971555537647671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,3584,0.039431111680136785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,4096,0.047672000196244985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,65536,1.0499413808186848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,1536,0.013400000002649097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,2048,0.0331857767370012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,3072,0.030400888787375555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,2560,0.02312177750799391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,2048,0.020936000678274367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,1536,0.01905866629547543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,768,0.015858666764365304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,512,0.006575111299753189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,256,0.005177777674463061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,1024,0.009972444838947719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,768,0.008486222061845992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,128,0.003560888684458203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,512,0.014878221684032016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,256,0.013637333280510373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,64,0.0027280000762807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,5120,32,0.0026506665680143568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,128,0.012997333374288348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,1024,0.01713244451416863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,64,0.01274311128589842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,5120,32,0.012849777936935425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,16384,0.22219555907779268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,16384,0.12364444467756484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,12288,0.17268533176845977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,10240,0.0809111131562127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,12288,0.10247644450929429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,10240,0.15051821867624918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,7168,0.09292355510923599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,8192,0.06733777787950304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,8192,0.10579199923409356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,7168,0.05912444326612684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,6144,0.08096444606781006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,65536,0.5397004551357693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,6144,0.053391999668545194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,5120,0.06806844472885132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,4096,0.055488000313440956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,3584,0.0492524438434177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,5120,0.04673066735267639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,3072,0.038260443343056574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,2560,0.03258311086230808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,65536,0.8290818002488879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,2048,0.012901333471139273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,4096,0.03539289037386576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,3072,0.024339555038346186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,1536,0.010496888723638324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,2560,0.022277333670192297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,1024,0.007735999921957652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,768,0.006653333289755716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,512,0.006955555743641323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,2048,0.020232889387342665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,3584,0.026501332720120747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,1024,0.016145777371194627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,768,0.015083556373914083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,256,0.004582222137186262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,128,0.0033235556135574975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,512,0.014083556003040738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,64,0.0027582223216692605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,1536,0.018241778016090393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,4096,32,0.002461333241727617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,128,0.011889778077602386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,64,0.01183199965291553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,256,0.0129466669427024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,4096,32,0.012098666694429187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,12288,0.14440266291300455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,16384,0.20027732849121094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,16384,0.11255110634697808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,12288,0.08728710810343425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,10240,0.11816622151268853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,7168,0.08336000310050116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,8192,0.09507733583450317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,8192,0.062112000253465444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,7168,0.0547653337319692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,10240,0.07366044653786553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,6144,0.07302221986982557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,65536,0.4990986718071832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,5120,0.06007288561926948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,4096,0.04928977621926201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,65536,0.7284657690260145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,3584,0.04375466704368591
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,6144,0.04894577794604831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,4096,0.02836266822285122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,3072,0.033647110064824425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,5120,0.043218665652804904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,2048,0.011701333026091257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,3584,0.025937777426507738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,1536,0.009841777384281158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,1024,0.007241778075695038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,2560,0.026943999860021803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,2560,0.02197333342499203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,2048,0.01998222205373976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,512,0.005232000102599462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,768,0.006339555399285422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,1024,0.015815999772813585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,256,0.004569777597983678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,3072,0.023967999551031325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,1536,0.01798488861984677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,768,0.014911111858155994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,32,0.0023839999404218462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,64,0.00272444449365139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,256,0.012418666647540199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,512,0.0138586668504609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,128,0.011777777638700275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,64,0.011406222151385413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3584,32,0.011735999749766456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,12288,0.12036266591813828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,16384,0.17109866937001547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,16384,0.10341155529022217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3584,128,0.003301333429084884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,12288,0.07874311341179742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,10240,0.1014817754427592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,8192,0.08279911014768812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,7168,0.0724488894144694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,10240,0.06735555330912273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,6144,0.06315288941065471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,7168,0.05002133382691277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,8192,0.061074667506747775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,65536,0.45519378450181747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,6144,0.04493688874774509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,5120,0.03235288792186313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,65536,0.6471315489874946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,5120,0.0536373323864407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,4096,0.027971555789311726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,2560,0.013081777426931592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,3072,0.03321955601374308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,4096,0.04273600048489041
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,3584,0.03802400165134006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,3584,0.02570488883389367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,2048,0.010459555519951714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,3072,0.023702222439977858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,2560,0.02184355590078566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,1536,0.008831111093362173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,768,0.005723555468850666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,1024,0.006640888750553131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,1536,0.01759466694460975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,512,0.00463733325401942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,1024,0.015544000599119397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,256,0.00432977784011099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,768,0.014666666587193808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,128,0.0030257778449190986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,64,0.002461333241727617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,2048,0.019509333703253005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,512,0.01353422221210268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,256,0.012349333200189801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,3072,32,0.002326222136616707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,128,0.011413333316644033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,64,0.011229332950380115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,3072,32,0.011332444846630096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,12288,0.09878489043977524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,16384,0.13107377952999538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,16384,0.0935208863682217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,12288,0.07067555851406522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,10240,0.08367821905348037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,8192,0.06807733244366117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,7168,0.059975114133622914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,6144,0.051721778180864125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,10240,0.062103999985588916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,7168,0.04533777634302775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,8192,0.05148888958825005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,65536,0.4253048896789551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,5120,0.0440906650490231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,65536,0.5280124346415201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,5120,0.0319164428446028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,4096,0.03647111190689935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,6144,0.03597777750757005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,3584,0.025684444440735713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,4096,0.02776088813940684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,3072,0.022251556317011516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,3584,0.0319217774603102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,2560,0.018806222412321303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,2048,0.014767999450365702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,1536,0.012200000385443369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,1024,0.008762666748629676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,2560,0.021498666869269475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,3072,0.02355466617478265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,1536,0.01758666667673323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,512,0.005585777676767773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,1024,0.015498666299713982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,2048,0.019371555911170114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,768,0.014275555809338888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,256,0.004077333129114575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,128,0.0033564445459180405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,64,0.002492444382773505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,32,0.0022693334354294673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,512,0.013282666603724161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,256,0.01218488895230823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,128,0.011249778171380361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,64,0.011114666859308878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2560,32,0.011211555865075854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2560,768,0.007274666594134436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,16384,0.10476266675525242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,12288,0.08007288641399808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,10240,0.0752426650789049
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,16384,0.08539288573794895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,12288,0.06412622001436022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,8192,0.05504711137877571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,7168,0.04857155680656433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,10240,0.056568887498643666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,65536,0.4298080073462592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,6144,0.04243110948138767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,65536,0.39876710044013125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,7168,0.0398231115606096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,8192,0.04721777637799581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,3584,0.011824000212881299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,5120,0.0318524440129598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,3072,0.021060445242457922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,6144,0.03556888964441087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,3584,0.025182222326596577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,4096,0.015069334043396844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,5120,0.03608977794647217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,2048,0.014342221948835584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,4096,0.027543110979927912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,1536,0.011595555477672152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,1024,0.008360889222886827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,3072,0.023427555958429973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,2560,0.017694221602545846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,2048,0.019148444135983784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,2560,0.021055999729368422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,768,0.0069448889957533936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,1536,0.017069333129458956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,1024,0.015145777000321282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,512,0.005362666729423735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,256,0.003951111187537511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,128,0.0029582221888833572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,768,0.014202665951516895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,512,0.013056000073750814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,64,0.0023688889212078517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,2048,32,0.0021280000607172647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,256,0.011952000359694162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,64,0.010741333166758219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,128,0.010966222319338055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,2048,32,0.01109155515829722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,16384,0.08036000198788114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,10240,0.051810665263069995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,12288,0.06165422333611382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,16384,0.07813866933186848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,12288,0.06037955813937717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,8192,0.042615112331178456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,7168,0.037751999166276716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,10240,0.0518488883972168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,6144,0.03298755486806234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,8192,0.04363110992643568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,5120,0.013495999905798169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,65536,0.3285377820332845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,7168,0.03971555497911241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,65536,0.3606444464789496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,4096,0.010563555690977307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,3584,0.010267555713653564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,5120,0.03174577818976508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,6144,0.035615109735065036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,2560,0.016626665989557903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,3072,0.009480000370078618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,4096,0.027453333139419556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,2048,0.013561777770519257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,3584,0.025262221693992615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,1536,0.010942222343550788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,1024,0.007895111209816402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,768,0.005372444374693765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,2560,0.021205332544114854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,1536,0.01718488832314809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,3072,0.023111111587948267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,1024,0.015035554766654968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,256,0.003658666792843077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,768,0.013968888256284924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,128,0.0026933333526055017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,2048,0.019137778215938143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,512,0.013021333350075616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,64,0.002320000073975987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,32,0.0019351111518012153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,256,0.011754666765530905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1536,512,0.005135111096832487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,128,0.01092266705301073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,64,0.010762666662534079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1536,32,0.01111288865407308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,16384,0.05486489004558987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,12288,0.04232088724772135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,10240,0.03600533472167121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,8192,0.016015110744370353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,16384,0.07601244582070245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,65536,0.2311617798275418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,12288,0.05990666813320584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,6144,0.012159110771285163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,7168,0.01331022216214074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,8192,0.04381600022315979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,10240,0.05173688795831469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,7168,0.039735111925337054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,4096,0.008925333619117737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,6144,0.035643554396099515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,65536,0.304912010828654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,5120,0.03164444367090861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,3584,0.01832444469134013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,3072,0.016211556063758004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,5120,0.012537777423858643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,4096,0.027280888623661462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,2048,0.01129066695769628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,3584,0.02530044482813941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,2560,0.01367644468943278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,1536,0.009014222357008193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,2560,0.021173333128293354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,1024,0.006639110959238476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,2048,0.019031110737058852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,768,0.005488889084921942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,3072,0.02332533399264018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,512,0.004280888785918553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,1536,0.017086222767829895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,256,0.0031591111587153543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,1024,0.015079110860824585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,128,0.002489777695801523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,64,0.002124444478087955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,768,0.014003554979960123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,512,0.012885332935386233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,1024,32,0.001985777790347735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,256,0.011828444070286222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,128,0.011017777853541903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,64,0.01071466671095954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,16384,0.042669332689709134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,1024,32,0.010994666980372535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,12288,0.03319022059440613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,10240,0.012297777665985955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,65536,0.171834667523702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,8192,0.010566222170988718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,16384,0.07607022258970472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,12288,0.05990666813320584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,7168,0.011191111471917896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,10240,0.05183022220929464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,6144,0.010410666465759277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,8192,0.04357599880960253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,5120,0.009083555804358589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,4096,0.012062222593360476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,65536,0.28540266884697807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,7168,0.03958844476275974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,6144,0.035515556732813515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,3584,0.01127644462717904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,5120,0.03153511219554477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,3072,0.00831911133395301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,2560,0.00792711145348019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,4096,0.027300443914201524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,2048,0.006700444552633498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,1536,0.008887999587588841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,1024,0.00656088896923595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,3584,0.025078222155570984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,3072,0.02314577831162347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,768,0.005389333185222413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,2560,0.020984888076782227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,2048,0.01885777711868286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,1536,0.01704266667366028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,512,0.004204444587230682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,256,0.0030097777230872046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,128,0.0023911111056804657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,1024,0.014944000376595391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,64,0.002001777705219057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,768,32,0.002208000048995018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,512,0.01276000009642707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,768,0.013791999883121915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,256,0.011672889192899069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,128,0.010934222075674268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,64,0.010871999793582492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,12288,0.010393777655230628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,16384,0.018256000346607633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,768,32,0.010978666444619497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,65536,0.10591644710964626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,10240,0.017047999633683097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,8192,0.014188443620999655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,7168,0.013444444371594323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,16384,0.07565155294206408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,12288,0.05989244249131945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,10240,0.051739556921852954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,6144,0.012200888660218982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,8192,0.043604443470637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,5120,0.01069866700304879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,4096,0.006451555424266391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,65536,0.2712053457895915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,7168,0.039573334985309176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,3584,0.006210666563775804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,3072,0.005868444426192178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,5120,0.031558222240871854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,2560,0.005608888963858287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,3584,0.025197333759731714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,3072,0.023081777824295893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,2048,0.006734222173690796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,4096,0.027281777726279363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,6144,0.035495112339655556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,2560,0.020967110991477966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,1536,0.008747555315494537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,1024,0.005867555737495422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,768,0.005249777601824866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,512,0.004067555483844546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,256,0.0029208889851967492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,2048,0.01887022289964888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,1536,0.016968000266287062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,1024,0.014933332800865173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,768,0.013776889277829064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,128,0.002334222197532654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,512,0.012707555459605323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,64,0.001973333251145151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,512,32,0.0021893332401911416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,256,0.011632888681358762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,16384,0.014128888646761576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,128,0.01109955542617374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,64,0.01074222226937612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,512,32,0.011021333436171213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,12288,0.008272888759771982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,65536,0.05236000153753492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,10240,0.010545777777830759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,16384,0.07559200127919515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,8192,0.00907733374171787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,7168,0.008867555194430882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,6144,0.007970666719807519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,12288,0.05990933047400581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,10240,0.051626665724648364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,8192,0.043523556656307645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,5120,0.0075795559419526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,7168,0.03961599866549174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,4096,0.007340444458855524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,3584,0.006949333681000604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,65536,0.27040089501274955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,6144,0.03556888964441087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,3072,0.0063475556671619415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,5120,0.03155555658870273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,2560,0.005803555664088991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,2048,0.005555555638339784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,1536,0.0046160001721647055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,3584,0.02502311103873783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,4096,0.027196443743175928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,3072,0.023059555225902136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,2560,0.02107200026512146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,2048,0.01888444523016612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,1024,0.00564533347884814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,768,0.004671999977694618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,512,0.00370755543311437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,1536,0.01689511040846507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,256,0.0028159999185138275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,1024,0.014735110931926303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,128,0.0022764443937275144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,768,0.013787555197874704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,64,0.002141333288616604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,256,32,0.002130666747689247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,512,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,256,0.011680000358157687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,128,0.011023999916182624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,16384,0.009345778160625035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,64,0.010690666735172272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,65536,0.025036444266637165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,256,32,0.010951999988820819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,12288,0.008239111138714684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,10240,0.007628444168302748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,8192,0.006847111301289664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,7168,0.0070497774415545994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,16384,0.07478400071461995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,12288,0.059593776861826576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,10240,0.05170489019817776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,6144,0.0064311110311084324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,8192,0.04341600007481045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,65536,0.2705004480150011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,5120,0.005935110979610019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,4096,0.005342222336265776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,3584,0.005260444349712796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,7168,0.03968977928161621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,6144,0.03538489010598924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,5120,0.03136533498764038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,3072,0.00517511119445165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,2560,0.004743111216359668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,4096,0.027165333429972332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,2048,0.004603555632962121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,3584,0.025154666768179998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,1536,0.004036444342798657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,3072,0.023002665903833177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,2560,0.02088711162408193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,1024,0.005493333356248007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,768,0.004534222185611725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,2048,0.0188044442070855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,512,0.003617777592606015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,256,0.0027022223091787765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,1536,0.01700622174474928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,128,0.002230222233467632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,1024,0.014891554911931356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,64,0.002151111140847206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,768,0.013734222286277346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,512,0.012705778082211813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,128,32,0.0021333334346612296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,256,0.011583999627166323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,65536,0.014316444595654806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,128,0.010979555547237396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,16384,0.0072266666425599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,12288,0.006790222393141852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,64,0.010609777437316047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,10240,0.006072888771692912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,128,32,0.011085333095656501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,8192,0.005591111050711737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,7168,0.005499555418888728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,16384,0.07530755466885038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,12288,0.05962577793333265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,6144,0.005203555441564984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,10240,0.05132888754208883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,8192,0.04331022169854906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,4096,0.004844444493452708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,7168,0.039111998346116804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,5120,0.0047173334492577445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,3584,0.004616888860861461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,65536,0.2676817841000027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,3072,0.0043333334227403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,6144,0.03525422347916497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,5120,0.031361778577168785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,2560,0.004469333423508538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,4096,0.02700622214211358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,2048,0.003871110992299186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,3584,0.024682665864626568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,1536,0.0037004442678557504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,1024,0.003527111063400904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,3072,0.022758222288555566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,768,0.004506666627195147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,2560,0.020829333199395072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,2048,0.01868888901339637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,1536,0.016833777228991192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,512,0.003584889074166616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,256,0.0028204443968004654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,1024,0.014519110321998596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,128,0.0022444443570242985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,64,0.0021191111041439907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,768,0.013581333061059317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,64,32,0.0021182222084866627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,512,0.012542222109105853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,256,0.011550222006109027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,128,0.010795555180973478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,65536,0.016195555528004963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,16384,0.006786666810512543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,64,0.010592000352011787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,12288,0.005724444571468566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,64,32,0.010933332973056369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,10240,0.0058577776783042485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,8192,0.0053377776510185665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,7168,0.005079111291302575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,12288,0.05920355849795871
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,16384,0.07597421937518649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,10240,0.051161779297722705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,6144,0.004635555462704765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,8192,0.04331466555595398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,5120,0.004403555558787452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,4096,0.0041253334946102565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,7168,0.03895644346872965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,65536,0.2688044442070855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,6144,0.0351582235760159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,3584,0.004832889056868023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,5120,0.031096001466115315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,4096,0.026712000370025635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,3072,0.004111999852789773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,2560,0.004396444393528832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,2048,0.003679111186001036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,3584,0.024693333440356787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,1536,0.003604444364706675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,3072,0.022729777627521094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,1024,0.003399111123548614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,768,0.005094222310516569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,2560,0.020626667473051283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,2048,0.018440888987647164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,512,0.003936000168323517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,1536,0.01662577854262458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,256,0.002773333340883255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,128,0.0022159999029503926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,1024,0.014614222778214348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,64,0.0021217777911159727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,768,0.013591110706329346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,80,32,32,0.0020977778153287037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,512,0.012574222352769641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,256,0.011243555280897351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,128,0.010840888652536603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,64,0.010614222122563256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,80,32,32,0.010840888652536603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,10240,1.2041698031955295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,12288,1.4548799726698134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,8192,1.6906053755018446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,16384,1.9060969882541234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,10240,2.0664516025119357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,8192,0.9326178232828776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,7168,0.7857786814371744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,12288,2.7285048166910806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,6144,0.7052524354722766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,7168,1.4378400378757055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,5120,0.5417537689208984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,6144,1.2739253573947482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,16384,3.2804987165662975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,4096,0.4420542187160916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,4096,0.8370782004462348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,5120,1.148007074991862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,3584,0.3865342140197754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,3584,0.7350044250488281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,2560,0.2890480094485813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,2048,0.43012534247504336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,2560,0.5326631334092882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,2048,0.25041066275702584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,1536,0.1835555500454373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,3072,0.358209768931071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,3072,0.7034568786621094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,1536,0.3581751187642415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,256,0.05606400304370456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,768,0.10447911421457927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,128,0.02172800070709652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,512,0.09326133463117812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,64,0.009923555784755284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,256,0.08886310789320205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,512,0.11801777945624457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,1024,0.24422044224209258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,1024,0.1396284500757853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,768,0.18816444608900282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,65536,32,0.007194666398896112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,64,0.09093333615197076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,128,0.08983644511964585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,65536,32,0.08645333184136285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,10240,0.30778667661878795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,16384,0.5105137825012207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,12288,0.3984079890780979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,12288,0.6830408838060168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,10240,0.515160878499349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,16384,0.8588533401489258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,7168,0.22146933608584932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,8192,0.4224666754404704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,6144,0.3260720041063097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,8192,0.2693217860327827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,6144,0.18668356206681994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,7168,0.4066346751319037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,5120,0.1514684491687351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,5120,0.26529333326551646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,4096,0.21505155828264025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,3584,0.18687910503811309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,3584,0.10757866832945083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,4096,0.1288835604985555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,3072,0.17624710665808785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,2560,0.14553599887424046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,2048,0.0692159997092353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,1536,0.08008266819847955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,3072,0.09837955236434937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,2560,0.08363378047943115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,1024,0.054677334096696645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,2048,0.11646044254302979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,65536,1.935972425672743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,1024,0.03852799865934584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,512,0.01607377827167511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,768,0.04510577850871616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,1536,0.05531644490030077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,256,0.005777777896987067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,768,0.01871200071440803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,128,0.0041991112132867174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,256,0.015326221783955893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,64,0.003679111186001036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,128,0.014208000567224292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,64,0.013908444179428948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,32,0.003572444534964032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,32,0.014335110783576965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,16384,512,0.01718844473361969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,16384,0.3731555673811171
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,12288,0.3043217658996582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,16384,0.6176008648342556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,12288,0.5174578030904134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,16384,65536,3.255607181125217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,8192,0.18678400251600477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,8192,0.31812622812059194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,10240,0.3892017735375299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,7168,0.276094224717882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,10240,0.25068622165256077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,5120,0.19788800345526802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,7168,0.1706142160627577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,6144,0.27077778180440265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,6144,0.14177600542704263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,5120,0.11891644530826145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,65536,1.3451253043280706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,4096,0.08722044361962213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,3584,0.13378489017486572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,3584,0.0775688886642456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,2560,0.058428446451822914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,3072,0.06778844197591145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,2560,0.09660444657007854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,3072,0.11459644635518391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,4096,0.17503378126356336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,2048,0.07721866501702203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,1024,0.04099644554985894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,1536,0.059157331784566246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,512,0.007129777636792924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,768,0.03147644466824002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,1024,0.01735377808411916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,2048,0.053210665782292686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,128,0.004422222160630756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,512,0.015463110473420886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,65536,2.4292373657226562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,768,0.016600888636377122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,64,0.003809777812825309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,128,0.013150222599506378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,32,0.003107555624511507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,1536,0.04248088929388258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,256,0.014180445008807711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,32,0.012968888713253869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,12288,256,0.0052471111218134565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,12288,64,0.013033777475357056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,12288,0.23740087615119088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,10240,0.1927680042054918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,16384,0.3121804396311442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,16384,0.5115288628472222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,12288,0.42873422304789227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,10240,0.3595306608412001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,8192,0.14692621760898167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,7168,0.22793867852952746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,6144,0.20057866308424208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,8192,0.2670373386806912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,6144,0.11940711074405247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,7168,0.13739200433095297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,4096,0.07513778077231513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,65536,1.132738643222385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,5120,0.164173338148329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,4096,0.14426667160458034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,5120,0.10033155812157525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,3072,0.1051937805281745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,2560,0.08824533224105835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,3584,0.07342666387557983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,2048,0.07162488831414117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,3584,0.12605867120954725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,2560,0.051199999120500356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,1536,0.05448266532686022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,1024,0.03463733196258545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,768,0.008608000146018134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,3072,0.06428088744481404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,512,0.006803555621041193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,1536,0.026183999247021143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,1024,0.017030222548378837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,256,0.0053031109273433685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,768,0.01593244406912062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,128,0.004127999974621667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,2048,0.04651644494798449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,64,0.0030382221771611106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,32,0.0029075555503368378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,128,0.012987555729018318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,512,0.014914666612943014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,32,0.012695111334323883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,256,0.013750221994188098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,10240,64,0.012682666381200155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,10240,65536,2.2590480380588107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,16384,0.26631198989020455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,16384,0.4142444398668077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,12288,0.3460666603512234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,12288,0.1993244489034017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,8192,0.1231528917948405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,8192,0.21566222773657906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,10240,0.2605733341640896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,10240,0.16176533699035645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,65536,0.8977173699273003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,7168,0.18488089243570963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,6144,0.16266043980916342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,6144,0.09270311064190334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,5120,0.13061866495344374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,4096,0.10441066821416219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,7168,0.11694399515787761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,5120,0.08037866486443414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,3072,0.0780222217241923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,2560,0.06541866726345487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,3072,0.054768890142440796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,3584,0.1013608906004164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,3584,0.06086044179068672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,2560,0.048024889495637685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,1536,0.04536711176236471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,4096,0.0731031099955241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,1536,0.02650844388537937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,65536,1.6079289118448894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,768,0.008719111482302347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,512,0.006333333336644703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,2048,0.05947644180721707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,2048,0.04187555445565117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,1024,0.023715555667877197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,256,0.004987555659479565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,768,0.023123555713229712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,128,0.003925333420435588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,1024,0.015319999721315173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,64,0.002918222298224767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,8192,32,0.002785777880085839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,512,0.022083555658658344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,256,0.020445333586798776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,64,0.019925332731670804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,128,0.01975644462638431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,8192,32,0.0192275560564465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,10240,0.2287751038869222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,16384,0.22989866468641493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,16384,0.3648506535424127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,12288,0.3045208983951145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,12288,0.18213155534532335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,8192,0.10432177782058716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,7168,0.15990844037797716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,8192,0.18915111488766143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,6144,0.15677332878112793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,7168,0.10119999779595269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,10240,0.1349902285469903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,5120,0.11224267217848037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,65536,0.7984648810492622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,6144,0.08163822359508939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,5120,0.07022755675845675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,3584,0.07942577865388659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,4096,0.10145155588785808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,4096,0.05929244226879544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,2560,0.05695377455817329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,2048,0.04629155662324694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,2560,0.0425839987066057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,3072,0.07583910889095731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,2048,0.031081779135598078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,3584,0.05860888957977295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,768,0.007089777953094906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,3072,0.048638221290376454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,512,0.005689777847793367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,1024,0.02237777743074629
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,768,0.021708443760871887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,256,0.00434666665063964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,1024,0.014096889230940076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,1536,0.0372977786593967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,512,0.020735111501481798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,1536,0.02383733292420705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,128,0.0037724446091387006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,32,0.00258577780591117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,64,0.0027537778433826235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,7168,65536,1.47289064195421
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,256,0.019136000010702345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,64,0.01831999917825063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,128,0.01830222209294637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,7168,32,0.018301332990328472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,12288,0.14088800218370226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,16384,0.19826132721371123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,12288,0.2650088999006483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,16384,0.3193964428371853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,10240,0.11184444692399766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,7168,0.13103466563754612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,6144,0.11501510938008626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,8192,0.09115199910269843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,7168,0.08103999826643202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,10240,0.2184222274356418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,8192,0.16011910968356663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,5120,0.09472799963421291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,5120,0.06273333231608073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,6144,0.07188266515731812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,65536,0.704468462202284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,3584,0.06733866532643636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,4096,0.053467555178536304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,4096,0.07977688974804349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,3584,0.04845244354671902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,3072,0.0436737769179874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,2048,0.039910223748948835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,2560,0.03371289041307237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,1536,0.030850665436850652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,3072,0.06511111391915216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,2048,0.023408888114823237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,65536,1.2274924384223092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,2560,0.048801776435640126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,1536,0.02184533410602146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,1024,0.012300444145997366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,512,0.0053102220926019884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,1024,0.020440888073709276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,768,0.019719999697473314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,128,0.003576000117593341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,768,0.009775111244784461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,256,0.004105777790149053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,64,0.0030533334033356774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,512,0.018836443622907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,256,0.017833777599864535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,128,0.017268444101015728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,6144,32,0.0025457777082920074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,32,0.0170462214284473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,6144,64,0.01752088963985443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,16384,0.26816267437405056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,12288,0.11292533079783122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,16384,0.15753244029151067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,12288,0.2023928827709622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,7168,0.1073804431491428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,10240,0.17912177244822183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,8192,0.0772613353199429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,8192,0.12954044342041016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,10240,0.10098755359649658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,65536,0.5990853309631348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,5120,0.07789599895477295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,6144,0.10728000269995795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,5120,0.05412444472312927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,4096,0.07043466965357463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,6144,0.061971558464898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,7168,0.07241155703862508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,4096,0.046034667227003306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,65536,1.0286453035142686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,3584,0.06219200293223063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,3072,0.05370311273468865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,2560,0.04054933455255296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,3584,0.042028443680869214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,3072,0.032123555739720665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,1536,0.019145778483814664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,768,0.009746666583749983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,1024,0.011943110989199745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,1536,0.021474666065639917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,1024,0.01982399986849891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,2560,0.0236151119073232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,512,0.005127111242877113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,128,0.0034391110142072043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,2048,0.02274755636850993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,768,0.019136889113320243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,256,0.0041946665280395085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,256,0.017159110969967313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,64,0.002603555512097147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,32,0.0024666666156715816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,128,0.016798221402698092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,32,0.016618667377365958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,5120,2048,0.037017779217825994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,64,0.01678400072786543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,16384,0.20084088378482393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,12288,0.16269511646694607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,10240,0.12153511577182347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,5120,512,0.01854311095343696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,16384,0.12057688501146104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,12288,0.10081244177288479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,6144,0.0837715533044603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,10240,0.07736888859007093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,8192,0.06458044714397855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,8192,0.10548710823059082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,7168,0.08509422010845608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,7168,0.058734224902258984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,65536,0.5007662243313259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,5120,0.06076711416244507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,4096,0.049675554037094116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,5120,0.04611555404133267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,3072,0.03826933436923557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,6144,0.05688889159096611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,3584,0.04344266653060913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,4096,0.03885066509246826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,65536,0.8251120249430338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,2560,0.03198400139808655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,2048,0.01687022215790219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,3072,0.023102222217453852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,2560,0.022065778573354084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,3584,0.027091556125217017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,2048,0.020424889193640817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,768,0.0075866662793689305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,1024,0.008991111483838823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,1536,0.013279111021094851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,256,0.004559111264016894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,512,0.005967111223273807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,1024,0.0184568895234002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,128,0.0034142221427626083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,768,0.017874666386180453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,256,0.015649777319696214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,32,0.002377777877781126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,4096,64,0.0028115556471877625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,128,0.015403555499182807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,64,0.015145777000321282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,32,0.015113777584499784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,512,0.017014222012625802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,16384,0.1900097794002957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,4096,1536,0.019779556327395968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,16384,0.10521244340472752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,12288,0.13832267125447592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,12288,0.08227644364039104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,10240,0.11439732710520427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,8192,0.09267555342780219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,8192,0.05848089191648695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,7168,0.07488622268040974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,10240,0.07597866985532972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,6144,0.0665795538160536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,7168,0.05316977699597677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,65536,0.49360354741414386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,65536,0.724063131544325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,5120,0.05418489045566983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,6144,0.047488888104756675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,3072,0.03405333227581448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,3584,0.04308355516857571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,4096,0.044180442889531456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,2560,0.023419555690553453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,5120,0.04158488909403483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,2048,0.01592266725169288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,4096,0.02565866708755493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,3584,0.02344088918632931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,1024,0.008834666675991481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,1536,0.013224889006879596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,1536,0.018731556004948087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,3072,0.02195377813445197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,2560,0.020865778128306072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,768,0.007491555478837755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,2048,0.01977688901954227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,128,0.003198222153716617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,256,0.004271111140648524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,512,0.0058266665372583605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,768,0.01683022247420417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,1024,0.01773955590195126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,512,0.016136889656384785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,32,0.002437333265940348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3584,64,0.002691555561290847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,256,0.015319110618697273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,128,0.014561777313550314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,64,0.014564444621404013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3584,32,0.014523555835088095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,12288,0.11726044283972846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,16384,0.09276533126831055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,16384,0.17867199579874674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,10240,0.09864266713460286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,12288,0.07350666655434503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,8192,0.0797520014974806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,7168,0.0657128890355428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,8192,0.05266311102443271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,7168,0.047951999637815684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,10240,0.062257779969109424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,6144,0.058479110399881996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,65536,0.3975164360470242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,5120,0.047353777620527476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,4096,0.03905511233541701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,6144,0.043057776159710355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,65536,0.6274435785081651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,3584,0.03415377934773763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,5120,0.029418667157491047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,3584,0.022460444105996027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,3072,0.029105779197480943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,2048,0.01459822224246131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,2560,0.019935111204783123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,2560,0.019945777124828763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,4096,0.023992000354660883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,1536,0.012168000141779581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,1024,0.008775999976529015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,2048,0.018961777289708454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,3072,0.021357332666714985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,1536,0.017829333742459614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,256,0.004024888906213972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,512,0.005507555686765247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,1024,0.01717066599263085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,128,0.0029928889125585556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,768,0.01633777717749278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,768,0.00737955586777793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,512,0.015406222807036506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,64,0.002324444345302052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,3072,32,0.0022417778770128884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,256,0.014404444230927361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,32,0.013732444081041547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,128,0.014006222287813822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,3072,64,0.013968888256284924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,16384,0.13020533985561794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,12288,0.09863822327719794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,10240,0.0834159983528985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,16384,0.08063022295633952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,12288,0.06451911396450467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,8192,0.06743999984529284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,7168,0.05659021933873495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,6144,0.050457778904173106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,10240,0.05442488855785794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,8192,0.046021332343419395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,65536,0.523456891377767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,7168,0.042078223493364125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,5120,0.042103111743927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,4096,0.03392444385422601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,3584,0.0284826656182607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,6144,0.02960977620548672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,5120,0.025557332568698462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,65536,0.3468168841467963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,2560,0.017792888813548617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,2048,0.014109333356221518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,3072,0.020829333199395072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,2560,0.019666666785875957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,4096,0.023149333066410486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,1536,0.011983111500740051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,3584,0.022121777137120564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,2048,0.018541332748201158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,1024,0.008358221915033128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,3072,0.02362222141689724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,512,0.005396444350481033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,256,0.003879111260175705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,768,0.01587288909488254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,768,0.007028444773621029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,512,0.015127999915017022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,128,0.0029768887907266617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,1024,0.016207999653286405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,1536,0.017464889420403373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,32,0.0021644443687465456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2560,64,0.002263111165828175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,256,0.013929777675204806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,128,0.01348888874053955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,64,0.013367111484209696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2560,32,0.01328888866636488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,12288,0.0801102254125807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,16384,0.10452533430523342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,10240,0.06729866398705377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,16384,0.06951910919613309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,12288,0.055529776546690196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,7168,0.046330667204327054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,8192,0.05504711137877571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,10240,0.04689777890841166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,8192,0.03801155421468947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,65536,0.4247902234395345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,6144,0.04073777794837952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,7168,0.02937600016593933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,65536,0.29156266318427193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,5120,0.03520444366667006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,3584,0.019511111908488803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,4096,0.023433778021070693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,3072,0.01699822147687276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,5120,0.02479733361138238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,4096,0.022422222627533808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,3584,0.021464889248212177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,2560,0.014268444644080268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,2048,0.01165511127975252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,6144,0.027215111586782668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,3072,0.020032889313167997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,1536,0.009677333136399588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,1024,0.006813333266311222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,768,0.0057706667317284485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,512,0.004509333521127701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,2560,0.018923555811246235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,256,0.0033777778347333274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,1024,0.015779554843902588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,2048,0.017838221457269456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,128,0.002801777794957161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,512,0.014352000421947904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,64,0.002439111057255003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,768,0.015214223000738354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,256,0.01329422245422999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,2048,32,0.0021439999755885866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,1536,0.016743110285864938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,128,0.013073777986897362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,64,0.012832000023788877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,2048,32,0.012872888810104795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,16384,0.08013866345087688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,12288,0.06181866592831082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,10240,0.05172977844874064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,16384,0.05871822436650594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,12288,0.04677955640686882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,8192,0.04257955484920078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,65536,0.324199120203654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,7168,0.041731556256612144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,10240,0.035840888818105064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,5120,0.02815999918513828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,6144,0.03671200076738993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,4096,0.020432000358899433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,8192,0.031038221385743883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,3584,0.018792000081804063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,65536,0.23622934023539224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,6144,0.026882666680547927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,7168,0.02901244494650099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,5120,0.024527110987239416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,3072,0.016336000627941556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,2560,0.013795555465751223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,3584,0.021039111746682063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,2048,0.011330666641394297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,1536,0.00923733330435223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,3072,0.01982933282852173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,2560,0.018581334087583754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,768,0.005552000055710475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,4096,0.02218222286966112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,2048,0.01737511157989502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,1024,0.006699555450015598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,512,0.004362666772471534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,1536,0.01643822259373135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,256,0.0032071111102898917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,128,0.002599999929467837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,1024,0.01517688896920946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,768,0.014672888649834527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,64,0.0021119999388853707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,512,0.014085332552591959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1536,32,0.0019066666977273093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,256,0.013110222087966071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,128,0.012685333689053854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,64,0.012572444147533841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,12288,0.03949244485961066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1536,32,0.012520000338554382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,16384,0.05114666620890299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,65536,0.20869867006937662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,8192,0.018360000517633226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,16384,0.04951644606060452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,10240,0.033302221033308245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,12288,0.040087110466427274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,10240,0.03555288910865784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,65536,0.1795724497901069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,7168,0.016568889220555622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,5120,0.013081777426931592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,7168,0.029040889607535467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,6144,0.014863999353514777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,6144,0.026737777723206416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,4096,0.011027555498811932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,3584,0.010479110810491772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,3072,0.010812444819344414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,8192,0.03091644578509861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,2560,0.009651555783218807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,2048,0.008240888516108194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,3584,0.020938666330443487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,3072,0.019641778535313077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,5120,0.024413333998786077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,4096,0.021958221991856892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,2560,0.018590221802393597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,1536,0.008933333059151968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,1024,0.006528888725572162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,768,0.005418666534953647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,2048,0.017460443907313876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,512,0.0041777777175108595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,256,0.0030826667530669104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,1536,0.01632355484697554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,1024,0.015141333142916361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,128,0.0024115554988384247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,768,0.014672888649834527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,512,0.013944889108339945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,64,0.0020275554723209804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,1024,32,0.0018240000224775737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,256,0.013059555656380124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,128,0.012666666673289405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,64,0.012512000070677863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,1024,32,0.012499555945396423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,16384,0.042564445071750216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,12288,0.03135910961363051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,10240,0.01777333352300856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,16384,0.04868355393409729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,65536,0.1604764461517334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,8192,0.014696000350846184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,7168,0.014408888088332282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,12288,0.03995022177696228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,65536,0.1586622264650133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,10240,0.03529244330194261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,5120,0.011027555498811932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,8192,0.030750221676296655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,6144,0.012443555725945367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,7168,0.028755555550257366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,6144,0.026558222042189703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,4096,0.009469333622190688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,3584,0.009113777842786577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,2560,0.007484444313579136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,4096,0.021681777305073206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,5120,0.02422399984465705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,3584,0.02069511181778378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,2048,0.006795555353164673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,3072,0.008382221890820397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,3072,0.019484443797005545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,1536,0.008824889030721452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,2560,0.018364444375038147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,768,0.005295111073387994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,1024,0.006435555716355641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,512,0.004102222207519743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,2048,0.017156443662113614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,1536,0.016114667057991028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,256,0.002979555477698644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,128,0.00239644447962443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,1024,0.014836443795098199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,768,0.01440177857875824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,512,0.013716444373130798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,64,0.0020275554723209804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,768,32,0.002179555594921112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,256,0.012958221965365939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,64,0.012488000094890594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,16384,0.01831022236082289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,32,0.012635555532243518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,12288,0.014535999960369535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,65536,0.09959910975562201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,768,128,0.012648888760142855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,10240,0.012802666260136498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,8192,0.00739822205570009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,7168,0.0070293330483966405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,16384,0.04839200112554762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,65536,0.1548808945549859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,12288,0.03984977801640829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,6144,0.010528888967302112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,10240,0.0354417794280582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,8192,0.03071199854214986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,5120,0.008553778131802877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,4096,0.00812533332241906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,7168,0.028770665327707928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,3584,0.007601777712504069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,3072,0.0070053330726093715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,6144,0.026525333523750305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,5120,0.02420355545149909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,2560,0.006434666613737742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,2048,0.005740444279379315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,1536,0.007720888488822513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,4096,0.021704889006084863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,3584,0.02075822154680888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,2560,0.018370666437678866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,768,0.004853333450025982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,1024,0.00572622236278322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,3072,0.019490665859646268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,1536,0.01611911091539595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,512,0.0038648889296584656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,2048,0.017101332545280457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,256,0.0029680000411139596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,1024,0.01497066683239407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,128,0.0023911111056804657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,768,0.014396443963050842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,64,0.0023351110931899813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,512,32,0.0021439999755885866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,512,0.013637333280510373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,256,0.0129120002190272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,16384,0.00757422215408749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,128,0.01274400038851632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,64,0.012595555848545499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,65536,0.053358223703172475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,512,32,0.012429333395428128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,12288,0.009167999857001835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,8192,0.007455999652544658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,10240,0.008462222086058723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,16384,0.04841599861780802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,7168,0.007166222565703922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,12288,0.039918220705456205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,10240,0.03540888759824965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,65536,0.154931558503045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,6144,0.00665244460105896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,8192,0.030694223112530176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,5120,0.006440889090299606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,7168,0.028799109988742407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,4096,0.00573600000805325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,3584,0.005619555711746216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,6144,0.026505778233210247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,3072,0.005135111096832487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,2560,0.005007110950019625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,5120,0.024137778414620295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,4096,0.02166044380929735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,2048,0.004504888835880491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,3584,0.0207013338804245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,3072,0.01942933268017239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,1536,0.004143110993835661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,1024,0.005609777652555042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,2560,0.018331555856598746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,768,0.0046471113132105935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,2048,0.017094223035706412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,512,0.0036968886852264404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,1536,0.016036444240146212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,256,0.002741333304180039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,128,0.0022355556074115965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,1024,0.014847111370828418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,64,0.002161777681774563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,768,0.014373333917723762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,256,32,0.002136888810329967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,512,0.01364177796575758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,256,0.012844444149070315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,16384,0.007816889219813878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,128,0.012537777423858643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,65536,0.022879110442267522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,64,0.012635555532243518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,12288,0.007267555428875818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,256,32,0.012322666744391123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,10240,0.007080000307824876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,8192,0.006491555521885554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,7168,0.005740444279379315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,16384,0.04830311073197258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,12288,0.03988088832961188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,65536,0.15464444955190024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,10240,0.035423109928766884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,6144,0.0052639999323421055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,8192,0.030693332354227703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,5120,0.005096000101831225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,4096,0.004606222112973531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,7168,0.02865333358446757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,3584,0.004913777940803104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,6144,0.02644266684850057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,3072,0.004490666505363253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,5120,0.02423911127779219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,4096,0.02160088883505927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,2560,0.0046977777447965415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,3584,0.02051822178893619
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,2048,0.003991999973853429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,1536,0.0038657776183552216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,3072,0.01942311061753167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,1024,0.003567111161020067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,768,0.004501333253251182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,2560,0.01832444469134013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,512,0.003588444242874781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,2048,0.01704266667366028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,1536,0.016022221909628976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,256,0.0026568888376156488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,128,0.002256888896226883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,1024,0.014904888139830695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,64,0.0021466666625605687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,768,0.014287999934620328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,128,32,0.002124444478087955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,512,0.013608000344700284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,65536,0.016329778565300834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,256,0.012678222523795234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,16384,0.006918222126033571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,128,0.012535110943847232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,64,0.01238666640387641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,12288,0.005872888697518243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,128,32,0.012516444755925072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,10240,0.005942222144868638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,8192,0.005407111098368962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,16384,0.048269334766599864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,7168,0.004892444445027245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,12288,0.03964622153176202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,10240,0.035190221336152815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,8192,0.0306000014146169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,6144,0.0047600000268883174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,65536,0.1539964410993788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,5120,0.004808889081080754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,4096,0.004257777912749184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,3584,0.004302222281694412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,7168,0.028346667687098186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,3072,0.004138666722509596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,6144,0.02623288830121358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,5120,0.023918223049905565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,2560,0.00442755553457472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,4096,0.021485333641370136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,2048,0.0037066667444176147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,3584,0.020464888877338834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,1536,0.003669333126809862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,1024,0.0034417777011791864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,3072,0.019367999500698514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,768,0.0035208890007601846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,2048,0.016922665966881644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,512,0.003583111282851961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,2560,0.018106665876176622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,1536,0.015984000431166757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,1024,0.01464000013139513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,768,0.014094221923086377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,256,0.0028417776856157514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,128,0.002230222233467632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,512,0.013312888642152151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,64,0.0021262222694026097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,64,32,0.0021155555215146807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,65536,0.016026667422718473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,256,0.012921777864297232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,16384,0.006712889091836081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,128,0.012423999607563019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,64,0.012255999777052136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,64,32,0.012275555895434486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,12288,0.005671111245950063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,10240,0.005767111149099138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,8192,0.0052737775776121355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,7168,0.004666666603750653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,16384,0.047873778475655444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,65536,0.15359289116329616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,12288,0.039466665850745306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,10240,0.0349333319399092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,6144,0.004442666553788715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,8192,0.030564443932639226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,5120,0.004307555655638377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,4096,0.004464000049564573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,7168,0.028316444820827905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,3584,0.004807999978462855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,3072,0.004052444464630551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,2560,0.0041128889554076726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,5120,0.023733332753181458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,6144,0.02591200007332696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,4096,0.021393777595625982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,3584,0.020336000455750358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,2048,0.0038897775941424896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,1536,0.003573333223660787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,3072,0.019127110640207928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,1024,0.0034346667428811393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,2560,0.017920000685585868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,768,0.00508622204264005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,512,0.003961777935425441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,1536,0.015743111570676167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,2048,0.01681422193845113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,1024,0.014422222971916199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,256,0.002817777709828483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,128,0.0021928888228204516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,768,0.014097778333557976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,512,0.01328888866636488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,64,0.0021226666867733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,64,32,32,0.0021235555824306277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,256,0.012700444294346703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,128,0.01222222215599484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,64,0.012198222180207571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,64,32,32,0.012239110966523489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,10240,1.191758261786567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,12288,1.4391857782999675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,16384,1.892231199476454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,8192,1.6815919876098633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,10240,2.0557617611355252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,8192,0.9189884397718641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,6144,0.6475111113654243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,7168,0.7802035543653699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,12288,2.717287911309136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,7168,1.4322835074530706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,5120,0.5745262040032281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,5120,1.0312479866875541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,6144,1.2713688744439018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,16384,3.270719952053494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,4096,0.4411875671810574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,3584,0.7299742168850369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,3072,0.3288035657670763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,3584,0.4061893357170953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,2560,0.5281973414950901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,2560,0.2992204560173882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,2048,0.23154221640692818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,2048,0.42560089959038633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,4096,0.9198737674289279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,1536,0.32532087961832684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,3072,0.6359057956271702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,1024,0.22126311726040312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,512,0.11786666181352402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,768,0.10288800133599176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,256,0.05243555704752604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,1024,0.1362533304426405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,1536,0.1936080058415731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,64,0.009573332965373993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,128,0.021583999196688335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,32,0.006755555669466655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,64,0.087281776799096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,128,0.08583022488488092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,65536,768,0.1837386687596639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,32,0.08875466717614068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,512,0.09177955653932358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,65536,256,0.08677511082755195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,16384,0.46653244230482316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,10240,0.3260337776607937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,12288,0.3933600054846869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,12288,0.7537546687655978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,10240,0.567380428314209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,8192,0.4190666675567627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,16384,0.9077332814534506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,8192,0.2666951020558675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,7168,0.23401154412163627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,6144,0.35386133193969727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,6144,0.19868355327182344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,7168,0.37285600768195254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,5120,0.29244889153374565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,3584,0.18651644388834634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,3584,0.10167466931872898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,5120,0.16432089275783962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,3072,0.15840089321136475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,65536,1.717135959201389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,4096,0.12757955657111272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,3072,0.08807733323838975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,2560,0.13032710552215576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,2048,0.10517244206534491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,4096,0.23302043808831108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,1024,0.054679112301932446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,2560,0.0832808878686693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,1536,0.08862666951285468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,2048,0.06835199726952447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,1536,0.05022044314278496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,768,0.046132445335388184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,256,0.005461333526505365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,128,0.004650666481918759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,512,0.016679111454221938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,256,0.01534400052494473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,512,0.01661422186427646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,64,0.003903999924659729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,32,0.0029884444342719186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,1024,0.03663911090956794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,768,0.017788444956143696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,128,0.01421866648726993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,64,0.014212444424629211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,16384,32,0.014224888549910652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,16384,0.3613297674391005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,16384,65536,3.2427493201361766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,12288,0.27858487764994305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,12288,0.573255115085178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,16384,0.6150248845418295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,8192,0.3192382123735216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,10240,0.38701510429382324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,10240,0.2467493216196696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,8192,0.1840746667650011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,7168,0.27128799756368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,7168,0.16784977912902832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,65536,1.3158861796061199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,6144,0.139829330974155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,6144,0.26905067761739093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,5120,0.20918756061130098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,4096,0.08713777860005696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,4096,0.15767555766635472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,3584,0.07660888963275485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,3072,0.11466933621300592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,5120,0.11901688575744629
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,3584,0.14898133277893066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,2560,0.096760888894399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,2048,0.07726666662428114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,3072,0.06758755445480347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,2560,0.05801244576772054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,1024,0.04085955686039395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,2048,0.05362488826115926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,1536,0.06577689117855497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,512,0.007048889166778988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,768,0.034760889079835676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,256,0.005112888912359874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,1024,0.01718222267097897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,1536,0.041371554136276245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,128,0.0041911109454101985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,768,0.01642311116059621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,512,0.015085332923465304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,64,0.003371555565132035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,256,0.013823111024167804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,32,0.0030986666679382324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,128,0.01312355531586541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,64,0.012979555461141797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,12288,32,0.013173333472675748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,12288,65536,2.4361866845024958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,16384,0.304819557401869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,12288,0.4274471071031358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,16384,0.5107199880811903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,12288,0.25243645244174534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,10240,0.3221546543969048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,10240,0.1920631064309014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,8192,0.28573067982991535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,7168,0.22599734200371635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,6144,0.20161867141723633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,6144,0.10717778073416816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,7168,0.1364648871951633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,8192,0.16030932797325984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,5120,0.1627617809507582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,4096,0.12889688544803196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,3584,0.06638755400975545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,65536,1.1106017430623372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,3072,0.10488622056113349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,3584,0.13418577777014837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,5120,0.10026311212115818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,2560,0.0883751114209493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,2560,0.05066133207745022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,1536,0.05468800001674228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,3072,0.0638186666700575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,1024,0.03742400142881606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,2048,0.04647733436690437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,4096,0.08270221948623657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,1536,0.025735111700163946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,1024,0.01680177781316969
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,2048,0.07868978050020006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,256,0.005069333232111401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,512,0.006688000013430913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,768,0.013150222599506378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,128,0.004039999925427967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,64,0.0030257778449190986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,512,0.014847111370828418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,32,0.0028844444702068963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,768,0.015641777051819693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,128,0.012846222354306115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,64,0.012649777862760754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,10240,65536,2.0009564293755426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,256,0.013711999687883588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,10240,32,0.012898666991127862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,12288,0.19417244858211943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,16384,0.2605404324001736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,10240,0.1584506697124905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,16384,0.41391910447014707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,12288,0.3448159959581163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,10240,0.2874373330010308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,8192,0.12007555696699355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,7168,0.1846524477005005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,7168,0.1059937808248732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,6144,0.16215644942389593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,5120,0.12974045011732313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,4096,0.10452000300089519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,6144,0.09131289190716213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,8192,0.236732456419203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,5120,0.07867466741138034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,4096,0.0665137767791748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,65536,0.8758168750339084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,3072,0.07793155643675062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,3072,0.05424088901943631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,3584,0.09066666497124566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,3584,0.06499733527501424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,2048,0.053152889013290405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,2560,0.0480915539794498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,2560,0.07318844397862752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,1536,0.04056977894571092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,2048,0.03975911272896661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,1024,0.011740444435013665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,768,0.00850222259759903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,512,0.006824888702895906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,256,0.004504888835880491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,1536,0.026135111848513286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,1024,0.023430221610599097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,768,0.022086222966512043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,64,0.0032142222755485107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,128,0.003644444462325838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,512,0.021800888909233943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,256,0.02016888890001509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,32,0.002733333243264092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,128,0.019899555378490023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,64,0.019279999865425956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,8192,32,0.02017155620786879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,8192,65536,1.7890845404730902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,16384,0.22627822558085123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,12288,0.30234045452541775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,16384,0.3631430996788873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,12288,0.17974932988484701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,8192,0.18742222256130645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,10240,0.131056891547309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,10240,0.22767199410332573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,65536,0.7740080091688367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,7168,0.1598622269100613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,8192,0.11343822214338516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,6144,0.14077688588036433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,7168,0.0912897785504659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,6144,0.08041599724027845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,5120,0.11231999927096897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,4096,0.0909608867433336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,3584,0.0796239972114563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,4096,0.05884977844026354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,5120,0.0757804446750217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,3072,0.06758222315046522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,65536,1.4010559717814128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,2560,0.05677066908942329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,2560,0.04583288894759285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,1536,0.03511910968356662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,2048,0.030460443761613634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,1024,0.007622222105662028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,3584,0.058055996894836426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,768,0.006675555474228329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,1536,0.022991110881169636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,2048,0.05062399970160591
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,1024,0.02156177825397915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,768,0.02130399975511763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,256,0.004317333300908406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,128,0.0034248888906505373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,512,0.005489777773618698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,512,0.020047111643685233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,3072,0.05208533340030246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,32,0.002970666728085942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,7168,64,0.0027342221389214196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,128,0.018333334061834548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,64,0.01851288808716668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,256,0.019132445255915325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,7168,32,0.01851733360025618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,16384,0.19232355223761666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,16384,0.3130853441026476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,10240,0.21584178341759575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,12288,0.14930666817559138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,12288,0.29021867116292316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,8192,0.15775022241804335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,10240,0.12030666404300266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,7168,0.14990133709377712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,6144,0.13338043954637316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,8192,0.08940444389979045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,6144,0.07100711266199748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,65536,0.6887582143147787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,7168,0.08868710862265693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,5120,0.10775111118952434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,3584,0.06815377871195476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,4096,0.05231111248334249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,5120,0.06736977895100911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,4096,0.08700799942016602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,3072,0.06489866971969604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,3584,0.04805155595143636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,2048,0.04454666707250807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,65536,1.1995742585923936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,3072,0.046359111865361534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,1536,0.03370577759212918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,1024,0.011770666473441653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,2048,0.02238133384121789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,768,0.00942666663063897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,2560,0.048736890157063804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,512,0.005103111267089844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,256,0.003947555604908201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,512,0.018580444984965853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,128,0.003516444315512975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,768,0.019509333703253005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,1536,0.021927111678653296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,256,0.017920888132519193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,2560,0.033486220571729876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,64,0.0026577777332729767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,1024,0.020234665936893888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,6144,32,0.002546666603949335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,32,0.017674666312005784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,64,0.01717688971095615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,12288,0.2149440050125122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,16384,0.15414399570888943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,6144,128,0.017249777913093567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,16384,0.2922968864440918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,12288,0.11104710896809895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,10240,0.15939377413855657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,10240,0.0919226672914293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,8192,0.14464710818396673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,8192,0.08364888694551255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,7168,0.10735644234551324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,65536,0.5820008913675944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,7168,0.0687662230597602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,6144,0.09496533208423191
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,5120,0.0872373316023085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,3584,0.05554133322503832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,6144,0.06652177704705133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,4096,0.07041511270734999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,3072,0.04790666699409485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,65536,1.0207466549343533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,5120,0.05343200100792778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,2048,0.03313422203063965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,4096,0.04909066690338982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,2560,0.04528266522619459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,1536,0.008525333470768398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,1024,0.011567999919255575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,3584,0.04102399945259094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,2048,0.02165511084927453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,3072,0.031680001152886286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,768,0.009215110705958473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,1536,0.020769778225156996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,512,0.004820444517665439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,256,0.0038328886859946777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,768,0.018739556272824604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,512,0.018033777674039204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,128,0.003059555465976397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,2560,0.02323466704951392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,256,0.01724444495307075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,32,0.002473777780930201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,5120,64,0.002599999929467837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,128,0.016763556334707472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,1024,0.01958400011062622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,64,0.016664889123704698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,5120,32,0.01717866626050737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,16384,0.21511822276645234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,16384,0.11614755789438884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,12288,0.09098311265309651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,12288,0.18203466468387178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,10240,0.12993155585394964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,8192,0.10502488745583428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,10240,0.07633066839641996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,8192,0.06377155250973172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,7168,0.0578133331404792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,6144,0.07565244701173571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,7168,0.09473599990208943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,65536,0.47937244839138454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,4096,0.04927466644181145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,6144,0.05170933405558268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,5120,0.06095555755827162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,5120,0.045490665568245776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,2560,0.03195644418398539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,3584,0.023982221881548565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,3072,0.03823022378815545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,3584,0.04848000076082018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,65536,0.8153688642713758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,4096,0.037851555479897395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,3072,0.022321777211295232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,2048,0.01609511176745097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,1024,0.008676444490750631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,2560,0.021367111139827307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,768,0.007380444142553542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,512,0.005550222264395819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,2048,0.02015199926164415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,256,0.0041422223051389056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,1536,0.01908533275127411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,1024,0.018029333816634286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,1536,0.01220444424284829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,128,0.0034328889515664843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,768,0.017575111654069688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,64,0.0024844445288181305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,512,0.01679644485314687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,4096,32,0.0023564443820052678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,128,0.015290665957662793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,64,0.015169777803950839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,32,0.015395555231306287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,4096,256,0.015826667348543804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,16384,0.1901644468307495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,12288,0.13617332776387533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,16384,0.10259555445777045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,10240,0.11373510625627305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,12288,0.08129777510960896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,8192,0.09244977765613133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,8192,0.05747999747594198
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,10240,0.07481777667999268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,7168,0.05224355392985874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,65536,0.4289608796437581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,6144,0.06672000222735934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,4096,0.04430933462248909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,6144,0.04674311147795784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,5120,0.05397599935531616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,4096,0.024287111229366724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,5120,0.041197333070966936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,3584,0.038760887251959905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,3072,0.03398311138153076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,7168,0.07964533567428589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,65536,0.7193457815382215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,2560,0.026890666948424444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,3584,0.022456889351209004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,1536,0.01218933363755544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,1024,0.008567999634477828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,768,0.00720355576939053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,3072,0.021470222208235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,2048,0.019444444113307528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,1536,0.018215111560291715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,512,0.005514666852023866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,256,0.004113777644104428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,1024,0.017542221479945712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,2560,0.020589333441522386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,128,0.0033537778589460584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,768,0.016512889001104567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,512,0.016064888901180692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,64,0.002424000038041009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,32,0.0022880000372727713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,256,0.01493866741657257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,128,0.014837332897716098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3584,2048,0.01457155578666263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,64,0.014410666293568082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,16384,0.16086667113833958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3584,32,0.014767110347747803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,12288,0.0720506641599867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,12288,0.11718755298190647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,16384,0.09162222014533149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,10240,0.09863111045625474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,8192,0.07977510823143853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,7168,0.06546133094363742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,10240,0.06648977597554524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,65536,0.37841955820719403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,7168,0.04696711235576206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,8192,0.051627556482950844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,6144,0.05844089057710436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,4096,0.0392177767223782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,65536,0.621979554494222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,3584,0.03415377934773763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,5120,0.028204444381925795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,5120,0.053429332044389516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,3072,0.029394666353861492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,3584,0.021777777208222285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,4096,0.023197333017985027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,2560,0.019867555962668527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,6144,0.042360888587103955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,2560,0.018431999617152743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,2048,0.018522666560279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,3072,0.0206675562593672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,1536,0.011596444580290051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,1024,0.008273777862389883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,768,0.0069253332912921906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,512,0.0053137776752312975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,256,0.0038862224254343244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,1536,0.01770400007565816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,2048,0.01442399952146742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,1024,0.016382222374280293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,768,0.015758223003811307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,128,0.002954666606254048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,512,0.015246222416559855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,32,0.0020595555090241963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,3072,64,0.0023048888478014204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,128,0.0140337778462304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,256,0.014522666732470194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,64,0.01388533330625958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,3072,32,0.014134221606784396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,10240,0.08356533447901408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,16384,0.07951288753085665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,12288,0.09833511379030015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,12288,0.06336622105704413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,16384,0.14144266976250544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,8192,0.06737955411275227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,7168,0.05702844593260023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,10240,0.053279111782709755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,6144,0.05050488975312975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,65536,0.32799111472235787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,8192,0.04546311166551378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,7168,0.04130666785769992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,65536,0.5178515646192763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,5120,0.0417191121313307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,6144,0.027871112028757732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,3584,0.026170666019121807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,5120,0.024666666984558105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,4096,0.02239822182390425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,3584,0.02142222225666046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,2048,0.014088888963063559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,4096,0.03372622198528714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,3072,0.021130666136741638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,2560,0.017323555217848886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,1536,0.011362666885058085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,2560,0.01941777765750885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,1024,0.008053333395057255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,768,0.006807999892367258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,2048,0.018165333403481376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,1536,0.017100445098347135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,512,0.005108444641033809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,1024,0.015975111060672335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,768,0.015490666031837463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,3072,0.020599111914634705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,512,0.014881778094503613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,256,0.0037519998020595978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,128,0.0029120000286234748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,64,0.0022817777676714794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2560,32,0.0020391111158662373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,256,0.013801777528391944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,128,0.01352533366945055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,64,0.013617777989970313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2560,32,0.01369599997997284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,16384,0.10475644138124253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,10240,0.06706400050057305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,12288,0.07978044615851508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,16384,0.06789688931571113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,12288,0.05388799972004361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,10240,0.04586933387650383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,8192,0.05474311113357544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,7168,0.04642044504483541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,65536,0.2737937768300374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,6144,0.04085155659251743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,65536,0.41928532388475204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,8192,0.03642311029964023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,5120,0.03460355599721273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,4096,0.023033777872721355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,7168,0.028499556912316218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,6144,0.026178666286998328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,3584,0.018565333551830716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,3072,0.016660445266299777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,4096,0.02166844407717387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,2560,0.01406222250726488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,2048,0.011439999772442712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,1536,0.009212444225947062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,3072,0.019779556327395968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,1024,0.006753777878151999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,5120,0.024125332633654278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,2048,0.017429333594110277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,768,0.005616000129116907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,3584,0.02064622276359134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,1536,0.016559110747443307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,2560,0.018567999203999836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,256,0.00327822214199437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,512,0.004361777669853634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,128,0.002748444469438659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,1024,0.015527110960748462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,768,0.014903111590279473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,64,0.0021884443445338146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,512,0.014110222458839417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,2048,32,0.001992000059949027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,256,0.01330044451687071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,128,0.012823999755912356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,64,0.01279022213485506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,2048,32,0.013086222112178802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,16384,0.08011466926998563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,12288,0.06148266792297363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,16384,0.05695288711123996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,10240,0.05170399943987528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,65536,0.32016534275478786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,12288,0.045612444480260216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,7168,0.04218222366438972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,8192,0.030163556337356567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,10240,0.03468711177508036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,6144,0.036158220635520086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,5120,0.02539644473128849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,65536,0.2190399964650472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,7168,0.02795377704832289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,6144,0.025959110922283594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,4096,0.02049955560101403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,3584,0.018331555856598746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,2560,0.013712888790501488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,5120,0.023731556203630235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,8192,0.04530577858289083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,4096,0.021373333202468023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,2048,0.011295999917719098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,1536,0.009109333157539368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,3584,0.020280000236299302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,1024,0.006588444527652528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,2560,0.01830755505296919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,2048,0.017157332764731515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,3072,0.01612355477280087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,3072,0.01942133406798045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,768,0.005494222044944763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,512,0.00424799985355801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,1536,0.016321778297424316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,256,0.0031315556002987754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,1024,0.01484444406297472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,64,0.002116444417172008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,768,0.014378666877746582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,32,0.0020248889923095703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,512,0.013850666582584381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1536,128,0.002609777781698439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,256,0.013030221892727746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,128,0.012796444197495779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,64,0.0125591109196345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1536,32,0.012821333275900947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,16384,0.05446666810247633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,12288,0.04217511084344652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,16384,0.04740266667471992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,10240,0.035903112755881414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,65536,0.22292089462280273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,8192,0.017638223038779367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,7168,0.015545777148670621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,65536,0.15983733865949842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,12288,0.03879288832346598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,10240,0.03433333171738519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,5120,0.012247111234400006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,6144,0.013546666337384118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,8192,0.030014223522610132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,4096,0.010312000082598792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,6144,0.025846223036448162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,3584,0.009707555174827576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,5120,0.023592000206311543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,2560,0.00941422250535753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,7168,0.02795733345879449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,4096,0.02106133268939124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,3072,0.008857777549160851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,2048,0.00815288888083564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,3584,0.020271999968422782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,3072,0.01928711103068458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,1536,0.008856000171767341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,2560,0.01830844415558709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,768,0.0052933332820733385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,1024,0.006060444646411472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,2048,0.01699555582470364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,1536,0.016007999579111736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,512,0.004101333518822988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,256,0.003003555453485913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,1024,0.014926221635606555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,128,0.0024337776833110386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,768,0.01424533294306861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,64,0.0020559999263948863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,512,0.013805333111021252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,1024,32,0.0022693334354294673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,256,0.012938666674825879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,128,0.012737777498033313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,64,0.012692444854312472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,1024,32,0.012582222620646158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,16384,0.042391998900307544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,12288,0.030823998981051978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,10240,0.01687111126052009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,65536,0.16399021943410239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,16384,0.046909332275390625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,65536,0.14792088667551676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,12288,0.03852444556024339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,8192,0.01388177772363027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,10240,0.03415733244684007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,6144,0.011895111037625207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,8192,0.02973244587580363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,5120,0.007039111521508958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,7168,0.013371555341614617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,7168,0.02772888872358534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,4096,0.00903200027015474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,6144,0.02570844524436527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,3584,0.008725333544943068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,3072,0.008016000191370646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,2560,0.007318221860461765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,2048,0.006629333313968446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,4096,0.020929776959949072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,3584,0.020206222931543987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,3072,0.019076445036464267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,5120,0.02334311107794444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,1536,0.00872444444232517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,2560,0.018072000808186002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,1024,0.005867555737495422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,2048,0.01683288812637329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,768,0.004950222041871813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,1536,0.015763555963834126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,512,0.004060444318585926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,256,0.0029786665820413162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,1024,0.01475911173555586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,768,0.014084445105658637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,128,0.0023742222951518162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,64,0.001993777851263682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,512,0.01352355546421475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,768,32,0.0021742222209771476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,256,0.012873777912722694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,128,0.012591111163298288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,16384,0.017672888106769986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,32,0.01257155587275823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,65536,0.10567555824915568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,12288,0.013571555415789286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,10240,0.012089778151777057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,16384,0.04677955640686882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,12288,0.03850311040878296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,768,64,0.012616888516479068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,65536,0.14772355556488037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,8192,0.0069368887278768755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,7168,0.006663110935025745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,10240,0.03408533334732056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,6144,0.010398222340477837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,5120,0.009256000320116678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,8192,0.029657777812745836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,4096,0.007994666695594788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,7168,0.027701333165168762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,3584,0.007431999676757389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,6144,0.02563200063175625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,3072,0.006919999917348226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,5120,0.023323555787404377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,4096,0.021075555019908484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,2560,0.006280888699822956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,3584,0.020192889703644645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,2048,0.005664000080691443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,1536,0.0076044441925154785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,3072,0.018944889307022095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,2560,0.018027555611398485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,1024,0.005686222265164058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,768,0.004780444420046276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,512,0.0037546666959921518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,2048,0.016799110505315993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,1536,0.01593955523437924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,256,0.0028231110837724474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,128,0.0023315555105606713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,1024,0.014659555421935188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,768,0.014017777310477363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,64,0.002179555594921112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,512,32,0.002124444478087955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,512,0.013461333182122974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,256,0.012871999707486896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,128,0.012639111114872826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,65536,0.05390222205056084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,64,0.012574222352769641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,12288,0.0064311110311084324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,16384,0.007256888680987888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,10240,0.008103999826643202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,16384,0.046682665745417275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,8192,0.007134222322040134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,12288,0.038458665211995445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,65536,0.14768977959950766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,7168,0.007480888730949826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,512,32,0.012595555848545499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,10240,0.034196443027920194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,6144,0.006861333217885759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,5120,0.006280888699822956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,4096,0.005642666584915585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,3584,0.005526222288608551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,7168,0.02776977750990126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,6144,0.025646222962273493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,3072,0.005069333232111401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,5120,0.023168888356950548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,8192,0.029828445778952703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,4096,0.020996444755130347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,2560,0.004926222066084544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,2048,0.004442666553788715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,3584,0.02015022271209293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,1536,0.004111999852789773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,3072,0.01899377836121453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,1024,0.005495999836259418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,768,0.004588444613748127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,2560,0.017975999249352347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,2048,0.016807110773192514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,1536,0.015856888559129503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,512,0.0036186666952239144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,256,0.0027102221631341507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,1024,0.014727110664049784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,128,0.002255111104912228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,64,0.0021422221842739317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,256,32,0.0021191111041439907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,768,0.014093332820468478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,512,0.013472889032628802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,256,0.012964444855848948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,128,0.012440000143316058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,65536,0.025806221697065566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,16384,0.007536888950400882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,64,0.01257599973016315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,12288,0.007035555938879649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,256,32,0.01274400038851632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,10240,0.006931555353932911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,8192,0.0058666666348775225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,7168,0.005657777604129579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,16384,0.04682488905058967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,12288,0.03846044341723124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,10240,0.03408622079425388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,6144,0.005175999883148405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,65536,0.14796000056796604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,5120,0.005038222091065513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,8192,0.029691557089487713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,4096,0.00453244439429707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,7168,0.027632888820436265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,3584,0.00479644454187817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,3072,0.004422222160630756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,6144,0.025587555434968736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,5120,0.02325600054528978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,2560,0.004638222356637319
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,2048,0.003940444439649582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,1536,0.00397866674595409
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,4096,0.020972443951500788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,3584,0.020022221737437777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,3072,0.0188071115149392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,1024,0.003589333345492681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,2560,0.0181351105372111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,2048,0.016672000288963318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,768,0.004507555729813046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,512,0.003589333345492681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,1536,0.015846222639083862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,256,0.0028106667515304354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,1024,0.014539556370841132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,128,0.0022355556074115965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,768,0.013910222384664746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,64,0.002157333410448498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,512,0.013413333230548434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,128,32,0.002104000084929996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,256,0.012960889273219638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,65536,0.01625777781009674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,128,0.012463111016485425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,16384,0.006893333461549547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,64,0.012527999778588613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,12288,0.005798222290145026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,128,32,0.0125608891248703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,10240,0.005928888916969299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,8192,0.005408000200986862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,16384,0.0466195543607076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,12288,0.03842577669355605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,65536,0.14700978332095677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,10240,0.033899555603663124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,7168,0.004880889008442561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,6144,0.004729777574539185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,8192,0.029683556821611192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,5120,0.004772444566090901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,7168,0.02753955622514089
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,3584,0.004847110973464118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,4096,0.004214222232500712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,6144,0.025575111309687298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,3072,0.004176889028814104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,2560,0.004429333325889376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,5120,0.02309333284695943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,4096,0.020789333515697055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,2048,0.0037253333462609183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,3584,0.019724445210562814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,1536,0.0036684444381131064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,3072,0.01891377733813392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,1024,0.003448888866437806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,768,0.003559999995761447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,2560,0.01793866687350803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,512,0.003956444561481476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,2048,0.016664889123704698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,1536,0.015579554769727917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,256,0.0028124445428450904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,1024,0.014276444911956787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,128,0.002222222172551685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,768,0.013970666461520724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,64,0.002143111079931259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,64,32,0.0021102221475707158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,512,0.01328000012371275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,65536,0.016082665986484952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,128,0.012339555554919772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,16384,0.006690666493442323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,256,0.012655999925401477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,64,0.01202044470442666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,64,32,0.01257155587275823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,12288,0.005662222289376789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,10240,0.005759999983840519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,8192,0.005218666460778978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,16384,0.04660355713632372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,7168,0.0046746668716271715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,65536,0.14710844887627497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,6144,0.004399999976158142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,10240,0.033618665403789945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,12288,0.03814399904674954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,8192,0.029520889123280842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,5120,0.004277333203289243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,4096,0.004496888981925117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,7168,0.02747822304566701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,3584,0.004456888884305954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,6144,0.02532622218132019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,3072,0.004193777839342753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,2560,0.0041413332025210066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,2048,0.0038542221817705366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,5120,0.023158222436904907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,4096,0.020768889122539096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,3584,0.01976799964904785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,1536,0.0035706667436493766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,1024,0.006165333506133821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,3072,0.018737778067588806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,768,0.004974222017659081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,2560,0.017646221650971305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,512,0.003926222109132343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,2048,0.01647555496957567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,1536,0.015665777855449252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,256,0.0027724444452259275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,128,0.0021946666141351065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,1024,0.014632888966136508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,768,0.013791999883121915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,64,0.0021137777302000257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,48,32,32,0.0020942222326993942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,512,0.013030221892727746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,256,0.012662221988042196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,128,0.012431999875439538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,64,0.012223111258612739
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,48,32,32,0.012496000362767113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,10240,1.1790711085001628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,12288,1.420863151550293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,16384,1.8606524997287328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,10240,2.04895994398329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,8192,1.6762569215562608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,8192,0.9124213324652778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,12288,2.704791174994575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,7168,0.8321119944254557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,6144,0.640559090508355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,7168,1.424814224243164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,5120,0.524528874291314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,16384,3.260895199245877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,6144,1.262896007961697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,4096,0.4189511140187581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,4096,0.8262293073866102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,5120,1.122344864739312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,3072,0.321852445602417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,3584,0.3965120050642226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,3072,0.6307039790683323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,2048,0.22396267784966362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,2560,0.5230808787875706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,2560,0.2921626567840576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,3584,0.8011048634847006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,1536,0.3206017812093099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,1536,0.18585067325168184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,512,0.1179342269897461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,1024,0.1230435503853692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,768,0.10180977980295818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,256,0.0587680008676317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,2048,0.4635778003268772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,1024,0.23733067512512207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,64,0.005991999887757831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,256,0.08122666676839192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,32,0.004373333520359463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,128,0.08162933588027954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,64,0.08407466941409641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,768,0.18037866221533883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,512,0.08607200119230483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,65536,128,0.021088000800874498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,65536,32,0.0821288890308804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,10240,0.3192319869995117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,12288,0.38190576765272355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,16384,0.4915751351250543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,10240,0.5540391074286567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,12288,0.7502035564846463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,16384,0.8075955708821615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,8192,0.26065066125657826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,7168,0.21731911765204537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,8192,0.41675466961330837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,7168,0.39890045589870876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,6144,0.18380267090267607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,6144,0.3499235577053494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,4096,0.12311289045545791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,3584,0.17893777953253853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,4096,0.22513066397772896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,5120,0.2861662175920275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,5120,0.15923821926116943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,3584,0.10887022150887384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,65536,1.6825902726915147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,2560,0.12558755609724256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,3072,0.16890754964616564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,1536,0.07657511366738214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,2560,0.08081689145829943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,2048,0.11342399650149876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,1536,0.049245334333843656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,3072,0.09450488620334202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,768,0.04432000054253472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,1024,0.057895110713111035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,768,0.016745777593718637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,512,0.012327111429638334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,256,0.005121777868933148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,1024,0.03658577799797058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,512,0.015591111448076038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,128,0.00490133340160052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,64,0.003126222226354811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,256,0.014313777287801107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,128,0.013289777768982781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,2048,0.06692800256941053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,32,0.0028880000528362063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,32,0.013037333057986366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,16384,64,0.013244444297419654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,12288,0.27289867401123047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,12288,0.5120879809061686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,16384,0.37715734375847715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,16384,0.6118737856547037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,10240,0.4247288968827989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,10240,0.2273813353644477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,16384,65536,3.1787456936306424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,7168,0.26933333608839244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,6144,0.23767643504672578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,7168,0.16625955369737414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,8192,0.19395644134945342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,8192,0.32755645116170246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,6144,0.1408053371641371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,5120,0.10521066851086086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,65536,1.2787858115302193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,4096,0.15505066182878283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,5120,0.21336266729566786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,4096,0.09446222252315944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,3584,0.1485040055380927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,3584,0.08426133129331802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,2560,0.09700977802276611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,2560,0.06301955382029216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,3072,0.1295751068327162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,2048,0.08720178074306911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,3072,0.07374488645129733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,2048,0.05235822333229912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,1024,0.0395066671901279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,768,0.033696889877319336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,512,0.0069982219073507525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,1536,0.04189244574970669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,1536,0.06589688857396443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,256,0.005078222188684675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,128,0.003539555602603488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,768,0.01484266585773892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,64,0.002797333316670524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,512,0.01389600005414751
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,32,0.0026693333768182327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,1024,0.01572533282968733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,128,0.012221333053376941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,256,0.013044444223244986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,64,0.012283555335468717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,12288,32,0.012355555262830524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,12288,65536,2.4013582865397134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,12288,0.24666754404703775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,16384,0.29581954744127065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,16384,0.5071822272406684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,10240,0.32024534543355304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,12288,0.47447999318440753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,8192,0.14203022585974798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,10240,0.18605866697099474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,8192,0.28391022152370876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,7168,0.13457155227661133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,6144,0.10455022255579631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,65536,1.0691777335272896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,5120,0.19141066074371338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,7168,0.2663404411739773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,4096,0.13948355780707464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,6144,0.2352293332417806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,3584,0.12202755610148112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,3072,0.09648799896240234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,3072,0.057371556758880615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,4096,0.07309066587024264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,5120,0.09797066450119019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,3584,0.07188710901472303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,2560,0.08106578058666654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,2048,0.0656213363011678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,1536,0.05532711082034641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,1024,0.03348533312479655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,2560,0.04972355564435323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,768,0.009468444519572789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,1024,0.014986667368147107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,512,0.0063271112740039825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,256,0.00480355570713679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,1536,0.02409422232045068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,2048,0.04546844296985202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,128,0.0032097777972618737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,256,0.012632889052232107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,128,0.012124444047609964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,512,0.013653332988421122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,64,0.002790222151411904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,32,0.0026293332791990707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,768,0.01425155500570933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,64,0.011964444484975604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,10240,32,0.01182933317290412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,10240,65536,2.175411648220486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,16384,0.25320445166693795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,10240,0.2570168972015381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,12288,0.34208356009589297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,12288,0.20184177822536895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,10240,0.15363289250267878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,16384,0.45826043023003477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,8192,0.21265155739254424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,7168,0.16724533504909941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,8192,0.1177262200249566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,6144,0.14714222484164768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,7168,0.10222133662965562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,5120,0.1181831094953749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,6144,0.09719377756118774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,3584,0.08336000310050116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,65536,0.847908443874783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,4096,0.10714577966266209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,5120,0.08299644125832452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,3072,0.08079822195900811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,2560,0.06051377455393473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,4096,0.06948266426722209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,2048,0.054682665401034884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,1536,0.03733511103524102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,1024,0.00998577806684706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,2560,0.04974755644798279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,3584,0.06298844681845771
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,3072,0.05221155616972181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,768,0.007822222179836696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,512,0.0058604445722368025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,1536,0.024736889534526404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,512,0.020265777905782063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,256,0.004548444516128964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,768,0.020370667179425556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,128,0.003584889074166616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,1024,0.021214221914609272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,65536,1.5948862499660914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,256,0.019352888067563374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,64,0.003145777723855443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,8192,32,0.002586666701568498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,2048,0.04204533497492472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,128,0.018833777970737882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,64,0.018560889694425795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,8192,32,0.01870933340655433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,12288,0.159987555609809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,16384,0.21892089313930937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,12288,0.3007226785024007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,16384,0.35950043466356063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,10240,0.12760978274875218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,10240,0.2505715423160129
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,8192,0.1002088917626275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,7168,0.14626399676005045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,65536,0.7495359844631619
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,8192,0.18573599391513398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,7168,0.08803733189900716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,5120,0.10399999883439805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,6144,0.14412532912360296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,6144,0.07737777630488078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,4096,0.08374044630262588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,3584,0.07340800099902682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,4096,0.056186669402652316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,3072,0.046253333489100136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,5120,0.07324000199635823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,3584,0.05589866638183594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,2048,0.043340444564819336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,3072,0.07100177473492093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,2560,0.04074666566318936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,1536,0.03685422076119317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,65536,1.3927635616726344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,1024,0.009098666409651438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,768,0.007445333732499017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,2048,0.03065599997838338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,512,0.005689777847793367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,1024,0.019592000378502738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,2560,0.0598542226685418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,256,0.004457777573002709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,1536,0.02130577796035343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,128,0.003731555408901639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,64,0.003076444483465619
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,512,0.01905333333545261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,7168,32,0.002471999989615546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,768,0.019620445039537217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,64,0.017694221602545846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,128,0.017344888713624742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,256,0.018187556001875136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,7168,32,0.017445334129863314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,12288,0.13132711251576742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,12288,0.2576977676815457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,16384,0.1853626701566908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,10240,0.1921724478403727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,16384,0.3439306683010525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,7168,0.12472532855139838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,10240,0.10735822386211819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,8192,0.086444444126553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,65536,0.6581022474500868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,6144,0.12417599889967178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,4096,0.07255466779073079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,8192,0.15652799606323242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,7168,0.08497599760691325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,5120,0.05918577644560072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,5120,0.08973066674338447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,4096,0.0500239994790819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,6144,0.07442577679951985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,2560,0.04649510979652405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,3584,0.0458568897512224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,2048,0.03760977917247348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,3584,0.07090666559007433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,3072,0.06137777699364556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,2560,0.03117777903874715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,1536,0.028597331709331934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,768,0.007276444799370236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,2048,0.020103999310069613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,1024,0.008711111214425828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,1536,0.019131556153297424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,512,0.005333333379692501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,65536,1.1849608951144748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,128,0.00332622230052948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,256,0.004275555411974589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,1024,0.0184568895234002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,64,0.0029075555503368378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,6144,32,0.0024293334119849736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,128,0.016379555066426594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,768,0.017668444249365065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,256,0.016543111867374845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,512,0.01735466718673706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,64,0.016145777371194627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,32,0.01605866683853997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,6144,3072,0.04471910993258158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,10240,0.15810578399234348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,12288,0.10686400201585557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,16384,0.1464328898323907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,12288,0.21259733041127524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,16384,0.2559502124786377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,8192,0.1288951105541653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,10240,0.09809244341320461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,6144,0.09255377451578777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,7168,0.10459644264645046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,8192,0.07338222530153063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,7168,0.06617155339982775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,6144,0.058408002058664955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,5120,0.08531555864546035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,4096,0.06095821989907158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,3584,0.05319111214743721
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,65536,0.6060924530029297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,3072,0.0463137792216407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,5120,0.0510062211089664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,4096,0.047240889734692044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,2048,0.03144888745413886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,3072,0.028712001111772325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,2560,0.04392710990375943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,2560,0.020796444680955675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,1024,0.009493333597977957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,1536,0.013777777552604675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,2048,0.01959288948112064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,768,0.007582222421964009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,1536,0.01865688959757487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,65536,1.0148400200737846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,512,0.005747555444637935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,3584,0.0431253347131941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,256,0.004310222135649787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,128,0.0029768887907266617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,1024,0.017673777209387887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,64,0.0027840000887711844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,768,0.017312000195185345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,512,0.01676266723208957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,5120,32,0.002416888872782389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,128,0.01625155574745602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,64,0.015963556038008798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,32,0.015916445189052157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,16384,0.19811999797821045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,12288,0.14926489194234213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,12288,0.08700178066889445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,5120,256,0.016336000627941556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,10240,0.1196204423904419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,8192,0.09650222460428874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,16384,0.12273600366380479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,65536,0.45311466852823895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,8192,0.060793777306874595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,7168,0.055303110016716846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,10240,0.08084533611933391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,6144,0.07393955522113375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,5120,0.06013955672581991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,65536,0.7520257631937662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,5120,0.0432240002685123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,7168,0.09424000316196018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,3584,0.04278399878078037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,4096,0.05608177847332425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,6144,0.05367822117275662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,4096,0.03464711043569777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,3072,0.0376782218615214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,3584,0.022670222653283015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,1536,0.009661333428488838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,2048,0.014595554934607612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,1024,0.0071937781241205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,3072,0.019256000717480976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,2560,0.018390221728218924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,768,0.0058782220714622075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,2560,0.031467556953430176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,256,0.003755555384688907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,1024,0.015654222832785714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,768,0.015430221954981485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,2048,0.01742755538887448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,512,0.014946666028764514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,128,0.002826666666401757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,512,0.00464088883664873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,64,0.002921777880854077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,4096,32,0.002623999905255106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,1536,0.01664444473054674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,256,0.01443288889196184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,128,0.014141332772043018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,64,0.014051554931534661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,4096,32,0.013949332965744866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,12288,0.13008356094360352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,10240,0.10523911317189534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,16384,0.19329867098066542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,12288,0.07724177837371826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,16384,0.09863022300932144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,10240,0.06566666894488864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,7168,0.07414044274224176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,8192,0.08516710996627808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,65536,0.40368445714314777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,7168,0.049828443262312144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,6144,0.06604977448781331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,8192,0.05457599957784017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,5120,0.05333422289954292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,6144,0.04437333345413208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,65536,0.6602515644497341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,4096,0.04388799932267931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,4096,0.02095111045572493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,3072,0.033379554748535156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,2048,0.011481777661376528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,3584,0.01902044481701321
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,3584,0.03794044587347243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,5120,0.042027556233935885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,1536,0.009753777749008602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,3072,0.018231999542978074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,1024,0.0069324444565508105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,2560,0.017522666189405654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,2560,0.024381332927280005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,768,0.00590133335855272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,256,0.00396799999806616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,2048,0.016572443975342643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,512,0.004411555412742826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,1024,0.015009777413474189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,768,0.014647111296653748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,128,0.0027786667148272195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,512,0.014493332968817817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,64,0.0028168888141711554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,1536,0.015800888339678448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3584,32,0.0025537777692079544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,256,0.01365244471364551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,64,0.01352533366945055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,32,0.013585777746306526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3584,128,0.013481777575280933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,16384,0.14780799547831217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,16384,0.08732444710201687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,12288,0.11169066694047715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,12288,0.06893244716856214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,8192,0.07371022303899129
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,10240,0.10213777754041885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,65536,0.35409066412183976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,8192,0.04882311158710056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,7168,0.0642684433195326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,10240,0.05816533168156942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,7168,0.044383111927244395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,65536,0.5681813557942709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,5120,0.04628711276584201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,4096,0.03793777690993415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,6144,0.0640853312280443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,6144,0.03979466689957513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,5120,0.025011556016074285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,3072,0.0292524430486891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,4096,0.018851555056042142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,2048,0.011404444773991903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,1536,0.010211555494202508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,2560,0.016350222958458793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,2048,0.01586577792962392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,3584,0.018228444788191054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,3584,0.03720622261365255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,3072,0.017447999782032438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,768,0.005662222289376789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,2560,0.016954667038387723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,1024,0.006796444455782573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,1536,0.015136889285511441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,256,0.0035208890007601846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,1024,0.014395554860432943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,512,0.0046862223082118565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,128,0.0027671110712819626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,64,0.0025271111064487034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,512,0.013559111290507846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,768,0.014121777481502958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,3072,32,0.00208177769349681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,256,0.01293511109219657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,128,0.013055110971132914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,64,0.012921777864297232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,3072,32,0.012780444489585029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,16384,0.11960089206695557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,12288,0.09029422203699748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,10240,0.07644177807701959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,16384,0.075162669022878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,12288,0.060235553317599826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,8192,0.06303022305170695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,10240,0.05056266652213203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,8192,0.042466666963365346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,7168,0.05452177921930949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,6144,0.049365331729253135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,5120,0.04098844528198242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,65536,0.305111116833157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,65536,0.4748817549811469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,7168,0.03880177934964498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,6144,0.024223110742039148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,4096,0.032776888873842024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,3584,0.02773155603143904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,5120,0.020023110840055678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,3072,0.018044445249769423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,4096,0.018339556124475267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,3584,0.017871111631393433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,3072,0.0169840008020401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,1536,0.010053333308961656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,2560,0.01647555496957567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,1024,0.006779555645253923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,768,0.005995555470387141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,2560,0.014294221997261047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,512,0.004688888788223267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,1024,0.013907555076811047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,2048,0.015418666932317944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,768,0.013581333061059317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,256,0.0032640000184377036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,128,0.0027875554644399216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,1536,0.014833778142929077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,64,0.002502222235004107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,512,0.013456000222100152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,256,0.012642666697502136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,32,0.0020088888704776764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,128,0.012387555506494312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,64,0.012456888953844706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2560,32,0.012343111137549082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2560,2048,0.011297778122954898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,16384,0.06336088975270589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,12288,0.07337422503365411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,16384,0.09870488776101007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,10240,0.061866667535569936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,8192,0.05013599991798401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,65536,0.2542568842569987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,65536,0.3847164312998454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,12288,0.0561751127243042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,10240,0.0432879991001553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,8192,0.03261155552334256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,7168,0.022936888866954382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,5120,0.03296000096533034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,7168,0.04432177874777052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,3584,0.010607110957304636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,6144,0.04362755682733324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,5120,0.019370666808552213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,4096,0.013258666627936892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,6144,0.020831111404630873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,2560,0.00868800034125646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,3072,0.009663110805882348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,3584,0.01722577710946401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,4096,0.017855111095640395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,2048,0.00831200016869439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,3072,0.016417778200573392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,1536,0.009012444151772393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,1024,0.006551110910044775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,768,0.005402666827042897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,2560,0.015786666009161208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,2048,0.01496444476975335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,1536,0.014194667339324951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,512,0.0042257776690853965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,1024,0.013432888521088494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,256,0.003132444495956103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,128,0.002465777720014254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,768,0.013091555900043912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,512,0.012708444562223224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,32,0.001975999938117133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,2048,64,0.0021217777911159727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,256,0.012157333393891653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,128,0.011923555698659686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,64,0.01181333346499337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,12288,0.058168000645107694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,16384,0.07332711087332831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,2048,32,0.011694221860832639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,16384,0.05334755447175768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,10240,0.04758844441837735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,12288,0.042633778519100614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,8192,0.038655112187067665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,65536,0.1972817712359958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,7168,0.03524622321128845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,10240,0.028324445088704426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,5120,0.011145778000354767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,6144,0.02994311187002394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,8192,0.023444443941116333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,7168,0.02217688825395372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,4096,0.009597332941161262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,3584,0.009282666775915358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,3072,0.00851644410027398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,65536,0.3361519972483317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,6144,0.020615110794703167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,4096,0.01757955551147461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,2560,0.009409777820110321
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,2048,0.006847111301289664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,3584,0.017063111066818237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,1536,0.00888177752494812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,5120,0.0192275560564465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,3072,0.016320889194806416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,1024,0.006500444478458828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,2560,0.015588444140222339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,768,0.0053280000057485365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,512,0.0041333333485656315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,1536,0.013944889108339945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,256,0.0030586665703190696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,1024,0.013297778036859302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,128,0.002529777793420686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,768,0.01293155550956726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,2048,0.014667555689811707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,64,0.002032888846264945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1536,32,0.0022604444788561925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,256,0.011944000091817645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,128,0.011761777930789523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,64,0.011504888534545898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,32,0.01143200033240848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,16384,0.04993155598640442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1536,512,0.01254755589697096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,12288,0.038535998927222356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,65536,0.20081688298119438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,10240,0.03292177783118354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,16384,0.04063555598258972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,8192,0.013767999907334646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,65536,0.13287466102176243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,6144,0.010610666539933948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,12288,0.02966488732231988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,7168,0.011944000091817645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,10240,0.026502221822738647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,5120,0.009505777723259395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,8192,0.023408888114823237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,7168,0.02216533323129018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,6144,0.020621332857343886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,3072,0.007182222273614671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,3584,0.007808888951937358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,4096,0.008160889148712158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,5120,0.01904088921017117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,4096,0.01761244402991401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,2560,0.006477333191368315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,3584,0.016982222596804302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,2048,0.005787555542257097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,1536,0.007798222204049428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,1024,0.0058346668051348785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,768,0.004928888960017098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,2048,0.014567999376191033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,2560,0.01551822159025404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,1536,0.01405244403415256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,3072,0.01627200014061398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,1024,0.013208889298968844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,512,0.0038693332009845306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,256,0.0030133333057165146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,128,0.0026640000028742682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,768,0.012809777425395118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,64,0.0023573332776625953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,1024,32,0.0021724444296624926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,256,0.011951111257076263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,512,0.012438221938080259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,64,0.011591999895042844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,32,0.011351111034552256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,16384,0.038819554779264666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,65536,0.14759111404418945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,1024,128,0.011719110939237805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,10240,0.010994666980372535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,12288,0.030155556069480047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,8192,0.00941777808798684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,65536,0.11856622166103786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,16384,0.0351128876209259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,12288,0.02927644385231866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,7168,0.010487111078368293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,6144,0.009476444787449306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,8192,0.023239110906918842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,5120,0.00833155545923445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,7168,0.021976888179779053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,4096,0.007428444094128079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,10240,0.026389333936903212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,3584,0.007024889190991719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,5120,0.018896889355447557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,6144,0.020357333951526217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,3072,0.006544888847404056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,4096,0.017501332693629794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,3584,0.016957332690556843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,2560,0.005917333480384614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,1536,0.007663110891977946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,3072,0.016168889072206285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,2048,0.00563466673096021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,2560,0.015414221419228448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,1024,0.005670222143332164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,2048,0.014522666732470194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,768,0.0047768888374169665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,256,0.002879111096262932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,512,0.0038364442686239877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,1536,0.013935110635227628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,1024,0.013165333204799227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,128,0.0023582221733199223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,64,0.0022088889446523455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,768,0.012877333495352002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,512,0.01239466667175293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,256,0.0120000003112687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,768,32,0.0021662221600612006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,128,0.011623111036088733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,16384,0.012251555919647217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,12288,0.00792266676823298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,64,0.01167555567291048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,768,32,0.01166311071978675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,65536,0.10026666853162979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,10240,0.007104888558387756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,8192,0.006543111056089401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,16384,0.03478044602606032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,12288,0.02938577863905165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,7168,0.006262222097979651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,65536,0.11129155423906113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,6144,0.0069422221018208405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,10240,0.026456000076399908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,5120,0.0064382221963670515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,4096,0.005716444303592046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,3584,0.005650666852792104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,7168,0.022006221943431433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,6144,0.020485333270496793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,3072,0.005170666509204441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,8192,0.023223999473783705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,5120,0.018942221999168396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,2560,0.005008000052637524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,4096,0.017345777816242643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,3584,0.0169493340783649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,2048,0.004554666578769684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,1536,0.0075253330998950535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,3072,0.016195555528004963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,1024,0.005633777628342311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,768,0.004680888934267892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,2560,0.015400888191329109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,2048,0.014523555835088095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,512,0.003703111161788305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,256,0.002809777855873108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,1536,0.013810666898886362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,1024,0.01314133322901196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,768,0.012853333519564735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,128,0.00226488895714283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,64,0.0021786666992637846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,512,32,0.002128888956374592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,512,0.012453333371215396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,256,0.011894221935007306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,128,0.011622221933470832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,64,0.011713777979214987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,65536,0.05065955718358358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,16384,0.006892444358931647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,12288,0.006182222316662471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,10240,0.007076444725195567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,65536,0.10612444745169745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,16384,0.03480089041921828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,8192,0.006527110934257507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,12288,0.02927644385231866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,512,32,0.011596444580290051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,7168,0.005709333138333426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,10240,0.02640266716480255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,6144,0.005261333452330695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,5120,0.005097777893145879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,4096,0.0046533333758513136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,8192,0.023190221852726404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,7168,0.021928000781271193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,3584,0.004906666775544484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,6144,0.020456888609462313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,5120,0.018936889039145578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,3072,0.004467555632193883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,2560,0.0047146665553251905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,2048,0.003998222036494149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,4096,0.017317333155208163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,1536,0.003990222182538775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,3584,0.016891555653678048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,3072,0.016114667057991028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,1024,0.00544799988468488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,768,0.004534222185611725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,2560,0.01556711064444648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,2048,0.014527999692493014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,512,0.003583111282851961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,1536,0.013958222336239286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,256,0.002700444517864121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,128,0.002228444442152977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,1024,0.01314222233162986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,64,0.0021457777669032416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,768,0.012755555411179861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,512,0.012451555165979596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,256,32,0.002114666625857353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,256,0.011973333027627734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,65536,0.020741333564122517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,128,0.011728000309732227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,16384,0.0069475554757648045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,64,0.011434666812419891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,12288,0.005883555445406172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,256,32,0.011391110718250275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,10240,0.005971555494599872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,8192,0.005450666778617435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,16384,0.03501333461867438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,7168,0.004885333279768626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,65536,0.10316977898279826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,12288,0.02941155433654785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,10240,0.02623644471168518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,6144,0.004783111313978831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,5120,0.0048062221871482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,8192,0.023111111587948267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,7168,0.021879111727078755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,3584,0.004861333303981357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,4096,0.004550222307443619
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,6144,0.020432889461517334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,3072,0.0041626666982968645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,5120,0.018975999620225694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,4096,0.017567111386193168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,2560,0.004443555656406614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,2048,0.0037137779096762338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,3584,0.01684977776474423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,1536,0.0036915557252036203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,3072,0.01612622208065457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,2560,0.015426667200194465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,1024,0.003456888927353753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,768,0.004524444540341695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,2048,0.014592889282438489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,1536,0.013770666387346057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,512,0.00360000009338061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,256,0.0028400001012616684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,1024,0.01296622223324246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,128,0.0022133334229389825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,768,0.012792000340090858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,64,0.0021386666016446217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,512,0.012311110893885294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,128,32,0.0021013333979580137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,256,0.011926222178671094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,65536,0.01610133383009169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,128,0.011702222128709158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,16384,0.006724444528420766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,64,0.011537777880827585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,12288,0.005688000056478713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,128,32,0.011549333731333414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,10240,0.005768888940413793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,8192,0.005274666680230035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,16384,0.03478666808870103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,65536,0.10305155648125543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,12288,0.029189334975348577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,10240,0.02613688839806451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,7168,0.004664888812435998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,8192,0.023063111636373732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,6144,0.004425777743260066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,5120,0.00433688900536961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,4096,0.004504888835880491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,3584,0.004461333155632019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,7168,0.021866665946112737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,6144,0.02036088870631324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,3072,0.00407911092042923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,4096,0.017232000827789307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,5120,0.018969777557584975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,2560,0.004152889053026835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,3584,0.016736888223224215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,2048,0.0038942222793896994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,1536,0.003590222034189436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,3072,0.015984889533784654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,768,0.00509066672788726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,1024,0.0034506666577524612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,2048,0.014533332652515836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,512,0.003948444293604957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,2560,0.015189333094490899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,1536,0.013820444544156393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,256,0.002817777709828483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,1024,0.012991111311647626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,768,0.012683555483818054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,128,0.002218666589922375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,64,0.0021191111041439907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,64,32,0.0020879999630981022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,512,0.01218311074707243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,65536,0.015860444969601102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,256,0.01180355581972334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,16384,0.006596444381607904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,128,0.011609777808189392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,64,0.011505777637163797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,64,32,0.01128088848458396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,12288,0.00555644432703654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,10240,0.005689777847793367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,8192,0.005186666631036335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,65536,0.09669244289398193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,7168,0.0052177777720822235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,16384,0.033230221933788724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,12288,0.027815110153622095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,10240,0.025176000263955858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,6144,0.004928888960017098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,5120,0.0046737777690092725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,8192,0.022428444690174524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,4096,0.004368888835112254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,7168,0.020999999509917364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,3584,0.004317333300908406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,3072,0.004127999974621667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,6144,0.019742222295867074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,2560,0.004064889003833135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,5120,0.01811733345190684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,4096,0.016973333226309884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,2048,0.003802666647566689
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,3584,0.016368889146380954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,1536,0.008392888638708327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,3072,0.015585778488053216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,1024,0.006174222048785951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,768,0.004998222407367495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,512,0.003913777983850903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,2048,0.014271999398867289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,1536,0.013559111290507846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,2560,0.01498044364982181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,1024,0.012845333251688214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,256,0.0027866665687825945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,768,0.01236177815331353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,128,0.002218666589922375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,64,0.0021031111892726687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,512,0.011935110721323224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,32,32,32,0.0020764445265134177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,256,0.011800888511869641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,128,0.011508444117175208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,64,0.011599111060301462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,32,32,32,0.011459555890825061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,10240,1.1739137437608507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,12288,1.4099288516574437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,8192,1.5270907084147136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,8192,0.9020587073432075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,16384,1.8501697116427953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,10240,1.86571651034885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,7168,0.7628142038981119
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,12288,2.4031687842475042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,7168,1.3105448616875541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,5120,0.5565422375996908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,6144,0.6801902453104655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,6144,1.1409848531087239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,4096,0.41167555914984805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,16384,2.979340447319878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,5120,1.0350418090820312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,3584,0.3923155466715495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,3072,0.5687075720893012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,3584,0.6615280045403374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,2560,0.26897422472635907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,4096,0.8259164492289225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,3072,0.33914756774902344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,1024,0.19462754991319445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,1536,0.1748595502641466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,2048,0.4124764336480035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,1536,0.2885475423600939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,2560,0.5201359854804145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,2048,0.23582667774624297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,512,0.0942231085565355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,768,0.1463599999745687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,128,0.012034667034943899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,1024,0.12851644886864558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,256,0.05239466826121012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,512,0.08276622162924872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,256,0.07910577456156413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,32,0.004261333495378494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,65536,64,0.005636444522274866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,64,0.08191288842095269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,768,0.10337511036131118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,128,0.08356889088948567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,65536,32,0.08420799838172065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,16384,0.4826515515645345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,12288,0.3692595428890652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,16384,0.7511279847886828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,10240,0.49494043986002606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,12288,0.6843048731486002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,8192,0.2308622201283773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,10240,0.3076631228129069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,7168,0.3298062218560113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,8192,0.4251217842102051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,7168,0.22170844342973497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,6144,0.2908959918551975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,6144,0.19203110535939535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,65536,1.6457600063747828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,3584,0.18260355790456137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,5120,0.2608364423116048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,4096,0.20781866709391275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,3584,0.10521155595779419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,5120,0.15735999743143717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,3072,0.14315911134084067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,2048,0.09487377934985691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,1536,0.07152533531188965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,3072,0.0980951123767429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,2560,0.1316088835398356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,4096,0.12820355097452799
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,2048,0.06863288746939765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,1024,0.04836622211668226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,2560,0.08426311280992295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,768,0.03685955537690057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,1024,0.03407199846373664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,512,0.012307555311255984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,768,0.01352355546421475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,1536,0.04948266678386264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,256,0.0047688889834615915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,128,0.0036151111125946045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,512,0.01257155587275823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,64,0.002975111206372579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,256,0.01129422254032559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,128,0.010861333045694562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,64,0.010455110834704505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,16384,32,0.01072088877360026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,32,0.002591111179855135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,16384,65536,2.9502942827012806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,12288,0.25788354873657227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,16384,0.3368559943305121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,12288,0.501164436340332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,10240,0.3823111057281494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,16384,0.6093084547254775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,8192,0.3105066617329915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,10240,0.23253954781426323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,7168,0.268979549407959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,7168,0.15048266781700984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,8192,0.18979644775390625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,6144,0.23568532201978895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,6144,0.12682844532860651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,5120,0.20949689547220865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,5120,0.1095199982325236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,3584,0.13397955894470215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,4096,0.09810044368108113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,3072,0.1251315540737576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,2560,0.09599910842047797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,3584,0.08640888664457534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,3072,0.06995644172032674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,65536,1.2450417412651908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,1536,0.05456088980038961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,1024,0.03921688927544488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,2048,0.08390666378868951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,768,0.02998133169280158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,2560,0.06355200211207072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,1024,0.01295111080010732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,1536,0.041024890210893415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,4096,0.16677867041693795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,256,0.0042408886882993905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,512,0.011783999701340994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,256,0.010799999866220685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,128,0.003301333429084884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,512,0.0075626663035816616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,2048,0.0518488883972168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,64,0.0030337776988744736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,768,0.012236444486512078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,32,0.002187555655837059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,128,0.010401777923107147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,64,0.009903999666372934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,12288,32,0.010127999716334874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,12288,65536,2.2134791480170355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,16384,0.281367990705702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,12288,0.39269065856933594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,12288,0.2345964378780789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,16384,0.5222080018785265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,10240,0.29571111996968585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,10240,0.17870755990346274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,8192,0.24226755566067168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,8192,0.156384891933865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,6144,0.18444177839491102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,6144,0.10762044456270005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,7168,0.2301644484202067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,5120,0.14935821957058376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,7168,0.13548710611131456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,65536,1.0430417590671115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,3584,0.10393689076105754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,5120,0.09939466582404242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,4096,0.1311688952975803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,3072,0.08955911133024429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,4096,0.08270577589670818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,3584,0.07378400034374662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,2560,0.07512978050443861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,2048,0.06054044432110257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,2560,0.04905688762664795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,1024,0.031381332212024264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,3072,0.06290577517615424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,1536,0.021500445074505273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,2048,0.04452088806364271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,1536,0.05106222298410204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,512,0.005706666658322017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,1024,0.012315555579132505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,768,0.01185155577129788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,256,0.003987555702527364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,512,0.011254222028785281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,128,0.003116444374124209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,65536,1.8459226820203993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,768,0.0073520003093613525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,128,0.00998133338159985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,32,0.002242666565709644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,256,0.010656000011497073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,10240,64,0.0028924445311228433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,64,0.009749333063761393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,10240,32,0.010102222363154093
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,16384,0.22501688533359102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,16384,0.3755013412899441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,12288,0.18455289469824898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,12288,0.3499200079176161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,10240,0.2644435564676921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,8192,0.19640621874067518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,7168,0.16624266571468777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,10240,0.14115466011895075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,8192,0.12583288881513807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,6144,0.16450577312045628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,6144,0.08870755301581489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,7168,0.11031288570827907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,5120,0.11810489495595296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,5120,0.0770124461915758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,4096,0.09568799866570367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,65536,0.8048728836907281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,3072,0.07189689079920451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,4096,0.06756177875730726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,2560,0.06040088997946846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,2048,0.0487333337465922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,3072,0.0477786660194397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,1536,0.037067555718951754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,2560,0.04035288757748074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,3584,0.08330755763583712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,768,0.0064071110553211635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,2048,0.03224710954560174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,1024,0.008755555583371056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,3584,0.059571557574801974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,256,0.003907555507289039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,1536,0.013813333378897773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,768,0.012145777543385824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,128,0.0032524443748924467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,512,0.00508266646001074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,64,0.002806222273243798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,512,0.011532444092962476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,1024,0.012464000119103326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,256,0.010820444259378644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,65536,1.4785306718614366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,8192,32,0.0023591110689772502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,128,0.01051644484202067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,64,0.0103511114915212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,8192,32,0.010988444089889526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,12288,0.14113956027560765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,16384,0.1932906707127889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,10240,0.20724177360534668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,12288,0.3012853463490804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,10240,0.12212533420986599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,16384,0.35171821382310653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,8192,0.17122222317589653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,7168,0.14560888873206243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,6144,0.1274577776590983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,5120,0.10373066531287299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,7168,0.08781333102120294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,65536,0.6976613468594022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,4096,0.08382399876912434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,3584,0.07325422101550631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,6144,0.08127644326951769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,5120,0.06668000088797675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,8192,0.10971199803882176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,4096,0.054879112376107105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,3072,0.06311200062433879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,2048,0.04306311077541775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,3072,0.040765331851111516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,2048,0.016436444388495553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,2560,0.03812266720665826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,1024,0.007679111427730984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,2560,0.05965333514743381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,768,0.006233777850866318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,1536,0.013057777451144325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,1536,0.032722665203942194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,3584,0.04721155431535509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,512,0.004975999808973736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,256,0.0038213332494099936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,1024,0.012075555821259817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,768,0.011808888779746162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,128,0.0029484445436133277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,32,0.0023431111541059283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,65536,1.2938977347479927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,256,0.010441777606805166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,512,0.011062222222487131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,128,0.010055111514197456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,7168,64,0.002752888947725296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,32,0.010485333700974783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,7168,64,0.010183999935785929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,12288,0.12159910466935898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,16384,0.2857288784450955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,16384,0.16373333666059706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,10240,0.17778400580088297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,12288,0.26583290100097656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,8192,0.1486942238277859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,10240,0.10617155498928493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,8192,0.08850933445824517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,7168,0.12457156181335449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,65536,0.6036364237467448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,5120,0.08921511305703057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,6144,0.06598222255706787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,7168,0.08164444234636095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,5120,0.05768089161978828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,4096,0.07241511344909668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,3584,0.06320977873272367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,6144,0.12410933441585964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,2560,0.04581422275967068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,3072,0.05487822161780464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,2048,0.03725422090954251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,3072,0.036165333456463285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,2560,0.019711110326978896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,65536,1.1087244881523979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,1536,0.02405333353413476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,2048,0.013171555267439948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,1024,0.007385777930418651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,3584,0.044491555955674916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,768,0.006504000061088138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,4096,0.0497920016447703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,512,0.00461777796347936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,256,0.003546666767862108
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,768,0.011345778074529437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,512,0.01070666644308302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,256,0.010144888526863521
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,32,0.0022728888110982045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,64,0.0027386666172080566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,128,0.009837333526876237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,1536,0.012493333054913415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,64,0.009740444521109263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,1024,0.011605333122942181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,6144,32,0.010036444498433007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,6144,128,0.0028817777832349143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,12288,0.10515644152959187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,16384,0.133605334493849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,12288,0.2224053276909722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,16384,0.24111466937594941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,8192,0.12331467204623753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,7168,0.10420355531904434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,10240,0.089355554845598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,65536,0.5079431003994411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,8192,0.0778133339352078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,10240,0.16642577118343777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,5120,0.07500800159242418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,6144,0.09230933586756389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,4096,0.06856711043251885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,3584,0.05312977896796333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,7168,0.06980088684293959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,3072,0.05204000075658163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,4096,0.04284444451332092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,65536,0.9266231324937609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,3072,0.019302222463819716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,5120,0.0514613323741489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,6144,0.060601777500576444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,2560,0.043327109681235425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,1536,0.009940444595283931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,2560,0.013408000270525614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,768,0.005573333137565189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,2048,0.012537777423858643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,1024,0.006747555401590135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,512,0.004379555583000183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,1536,0.01185066666867998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,3584,0.033646222617891096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,256,0.0034888887570963967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,128,0.00282222218811512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,768,0.010875555376211802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,256,0.009926222264766693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,64,0.002699555622206794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,1024,0.011140444212489657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,2048,0.03538844320509169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,128,0.009724444813198512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,5120,32,0.0022897778285874263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,512,0.010640000303586325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,64,0.00962577760219574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,5120,32,0.00997511131895913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,16384,0.10947289069493611
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,16384,0.1942666636572944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,12288,0.17825422022077772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,10240,0.11863466103871663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,12288,0.09172177977032131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,8192,0.10014400217268203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,7168,0.08419822321997748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,10240,0.07123644484413995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,65536,0.41594934463500977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,8192,0.06259822183185153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,6144,0.07434844308429293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,5120,0.06038755840725369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,7168,0.056794669893052846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,4096,0.05584266450670031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,3584,0.04292444388071696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,6144,0.045223110251956515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,5120,0.039192001024881996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,3072,0.03787822193569607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,2048,0.013550221920013428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,3584,0.014287999934620328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,65536,0.7431919839647082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,3072,0.013381332986884646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,1024,0.0063893331421746155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,2560,0.0313937763373057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,2048,0.011917333636018964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,1536,0.01129422254032559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,768,0.0053324442770746015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,4096,0.02847911251915826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,2560,0.012724444270133972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,1024,0.010748444332016839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,768,0.01054933336046007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,256,0.0034257777863078644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,512,0.01013422260681788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,128,0.0028204443968004654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,64,0.00272088891102208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,512,0.004386666748258802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,1536,0.009445333646403419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,128,0.009345778160625035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,4096,32,0.0022799999763568244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,256,0.009655999640623728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,32,0.009714666340086196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,4096,64,0.0092613332801395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,16384,0.17034667068057588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,16384,0.09642133447859023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,12288,0.13952799638112387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,10240,0.10455822282367283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,12288,0.0752097765604655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,8192,0.08739999930063884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,10240,0.06359822220272489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,7168,0.07428622245788574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,6144,0.06618577904171415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,7168,0.046262221203909985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,65536,0.40179377131991917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,8192,0.050716443194283374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,6144,0.040623999304241605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,4096,0.04394222299257914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,3584,0.03817777832349142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,5120,0.03523822294341193
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,4096,0.01418133411142561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,3584,0.013665777941544851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,2560,0.028068443139394123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,2048,0.010377777947319878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,65536,0.6535449028015137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,3072,0.01295733369059033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,3072,0.03340444299909804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,1024,0.006287999865081575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,2048,0.011728888584507836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,2560,0.012312889099121094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,768,0.005254222287072076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,1536,0.008769777913888296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,1536,0.011181332998805575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,512,0.004389333228270213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,256,0.0034026667061779234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,5120,0.05985955397288004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,128,0.0027662221756246355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,64,0.0027093332674768236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,512,0.00980088859796524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3584,32,0.002430222307642301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,1024,0.010405333505736457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,128,0.00904977818330129
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,256,0.009389333426952362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,768,0.010346666806273991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,64,0.009104000197516548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3584,32,0.009619555539555019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,16384,0.14762666490342882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,16384,0.08453955252965291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,12288,0.12039821677737766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,8192,0.07644089063008626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,10240,0.0995946658982171
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,10240,0.055416001213921436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,12288,0.07183733251359728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,65536,0.3198631074693468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,6144,0.05810044209162394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,8192,0.04495288928349813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,5120,0.04697155621316698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,65536,0.5590391159057617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,7168,0.04074577821625604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,4096,0.03819200065400865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,6144,0.03606044583850437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,7168,0.06530755758285522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,3072,0.02960977620548672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,3584,0.03721155391799079
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,3584,0.013459555804729462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,2560,0.014644443988800049
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,4096,0.013919111755159167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,2048,0.010294222169452244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,1536,0.008738666772842407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,3072,0.012792888614866467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,1024,0.006434666613737742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,2048,0.011550222006109027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,768,0.005456888841258154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,1536,0.011026666396194033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,512,0.004308444344335132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,1024,0.010298666854699453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,256,0.003567111161020067
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,5120,0.01867377758026123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,768,0.010059555371602377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,2560,0.012248000337017907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,512,0.009841777384281158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,128,0.002769777758253945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,64,0.002696000039577484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,3072,32,0.0023839999404218462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,256,0.009345778160625035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,64,0.008754666480753157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,128,0.009128889275921715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,3072,32,0.009152000149091085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,16384,0.12425243854522705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,12288,0.10194755262798733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,16384,0.07240800062815349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,10240,0.07772355609469943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,12288,0.05690044164657593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,8192,0.0661262207561069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,65536,0.27256978882683647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,7168,0.055192000336117215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,6144,0.04907200071546766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,8192,0.03926488757133484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,7168,0.035460445615980364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,10240,0.04700977934731377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,5120,0.04075288772583008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,4096,0.032828443580203585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,65536,0.4683244493272569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,5120,0.015187554889255099
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,3584,0.02424266603257921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,3072,0.0177208897140291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,2560,0.013367999758985309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,4096,0.013886222408877479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,1536,0.008453333543406593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,3072,0.012681778106424542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,2048,0.009820444716347588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,6144,0.02055999967787001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,2560,0.012125333150227865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,768,0.005593777944644292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,2048,0.011314666933483548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,512,0.00424177779091729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,1536,0.01088711122671763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,3584,0.013259555730554791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,1024,0.010264000131024255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,256,0.0033280000918441345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,128,0.00327822214199437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,512,0.009597332941161262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,1024,0.0063564446237352155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,768,0.010013333625263637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,32,0.0023253332409593794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2560,64,0.00267999991774559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,256,0.00925688859489229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,128,0.009024000002278222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,64,0.008825778133339351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2560,32,0.009172444542249044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,12288,0.0831786659028795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,16384,0.10537066724565293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,16384,0.059976001580556236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,10240,0.06672088967429267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,12288,0.04784355560938517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,8192,0.054678221543629967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,65536,0.22327378061082628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,10240,0.0396035545402103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,65536,0.37622488869561094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,7168,0.0463013317849901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,8192,0.02422488894727495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,6144,0.0413991113503774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,5120,0.03446844551298354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,6144,0.016040000650617812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,7168,0.01773866679933336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,5120,0.015023999743991427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,3072,0.013777777552604675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,4096,0.02237333357334137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,4096,0.013669333524174161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,2048,0.00960888879166709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,3584,0.016346666547987197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,3072,0.012523555921183692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,1536,0.008057778080304464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,2560,0.01202222208182017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,2560,0.012430222498046027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,2048,0.01127644462717904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,1024,0.0058497778243488735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,3584,0.013188444077968597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,768,0.004957333207130432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,1536,0.010793777803579966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,256,0.0033964444365766314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,512,0.003970666478077571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,1024,0.010076444182131026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,128,0.0026613333159022858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,64,0.002601777720782492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,768,0.009841777384281158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,512,0.009502222140630087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,256,0.0090844440791342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,2048,32,0.0023022221608294383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,128,0.008952000074916417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,64,0.008813333180215623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,2048,32,0.00903200027015474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,16384,0.07722755273183186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,12288,0.05760888920889961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,10240,0.04781422350141737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,12288,0.03864977757136027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,16384,0.048710223701265126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,8192,0.041231112347708807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,10240,0.02104088829623328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,65536,0.174582216474745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,7168,0.04205866654713949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,65536,0.2841342290242513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,6144,0.03584266702334086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,8192,0.018069333500332303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,5120,0.027387556102540758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,7168,0.017177777157889474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,3584,0.015159999330838522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,6144,0.016000888413853116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,5120,0.014878221684032016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,2560,0.011557333171367645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,4096,0.013570666313171387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,3584,0.013053333593739403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,3072,0.013437333206335703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,4096,0.016961778203646343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,2048,0.009521777431170145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,3072,0.012473777764373355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,2560,0.01197866681549284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,768,0.004839111119508743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,1024,0.005882666756709416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,512,0.0038622220357259116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,1536,0.007694222033023834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,1536,0.010729778144094678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,1024,0.010002666877375709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,256,0.003059555465976397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,768,0.009849777652157677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,128,0.0028968888024489083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,2048,0.011184000306659274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,64,0.0024071110205517872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,512,0.009448889229032729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1536,32,0.002238222294383579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,256,0.009094222552246517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,128,0.008850666383902231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,64,0.008778666456540426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1536,32,0.009192888935407003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,16384,0.052173333035575024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,12288,0.03828000028928121
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,10240,0.03297955460018582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,65536,0.19275289111667207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,16384,0.030962665875752766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,8192,0.01089333328935835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,12288,0.022711111439598933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,65536,0.12030843893686931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,7168,0.008893333375453949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,10240,0.020384889509942796
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,6144,0.008243555823961893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,5120,0.007608888877762689
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,4096,0.006653333289755716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,8192,0.01809155609872606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,6144,0.01590488851070404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,7168,0.017094223035706412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,3584,0.00654755574133661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,5120,0.0148026661740409
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,3072,0.01316088851955202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,2560,0.011354666617181567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,2048,0.00942311104800966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,1536,0.007589333587222629
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,3584,0.013007111019558378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,3072,0.012472888661755456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,2560,0.01197866681549284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,2048,0.011152889165613385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,4096,0.013558222187889947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,768,0.004729777574539185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,1024,0.005626666463083691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,1536,0.010573333336247338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,512,0.003793777690993415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,256,0.0029093333416514923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,1024,0.009949333137936061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,64,0.0022719999154408774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,512,0.009463999834325578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,128,0.0023564443820052678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,256,0.00907733374171787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,768,0.009820444716347588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,1024,32,0.0021644443687465456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,128,0.008905777500735389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,64,0.008758222063382467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,16384,0.03974666529231601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,1024,32,0.0090515555606948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,12288,0.029548443026012842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,10240,0.01018666641579734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,65536,0.15148799949222141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,16384,0.027066666218969557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,8192,0.009165333376990424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,65536,0.10651822222603692
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,7168,0.008792888787057664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,10240,0.020255999432669747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,12288,0.022633777724372015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,5120,0.0075662218862109715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,7168,0.016929777132140267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,6144,0.008007110820876228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,4096,0.006596444381607904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,6144,0.015801777442296345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,3584,0.006027555714050929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,5120,0.014799111419253878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,8192,0.01795644395881229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,3072,0.005474666754404704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,2560,0.005440000030729506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,4096,0.013374221821626028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,3584,0.01294222225745519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,1536,0.007459555235173967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,3072,0.01240799989965227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,2560,0.011844444606039258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,2048,0.004798222333192825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,1024,0.00554577757914861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,768,0.004582222137186262
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,2048,0.011167999770906238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,512,0.003660444584157732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,1536,0.010562666588359408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,256,0.0028906667398081887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,128,0.002263111165828175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,1024,0.009882666998439366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,768,0.009743111001120673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,512,0.009382222261693742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,64,0.002209777840309673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,768,32,0.0021555556191338436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,256,0.009003555609120263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,128,0.008966222405433655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,16384,0.014021333720948962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,64,0.008818666968080733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,768,32,0.009128889275921715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,12288,0.010187555518415239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,65536,0.10164711210462783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,10240,0.008823110825485652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,8192,0.007800000409285228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,65536,0.09547289212544759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,16384,0.0270506673389011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,12288,0.02258311046494378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,7168,0.007409777906205919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,10240,0.020258666740523446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,6144,0.006844444407357111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,5120,0.006304889089531369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,8192,0.017989334132936265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,4096,0.005488889084921942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,7168,0.01703555550840166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,3584,0.00546488869521353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,6144,0.015848888291252982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,3072,0.005008888741334279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,5120,0.014673777752452426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,2560,0.005062222066852781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,2048,0.004490666505363253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,1536,0.004423111263248655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,3072,0.01237511138121287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,3584,0.013032889200581444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,4096,0.013438222308953604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,2560,0.011890666352377998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,1024,0.005505777895450592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,768,0.004544888933499654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,512,0.003636444608370463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,1536,0.0105253333846728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,2048,0.011136888629860349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,256,0.0027679999669392905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,128,0.0021866667601797315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,1024,0.009884444375832876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,768,0.009714666340086196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,64,0.002157333410448498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,512,32,0.0020728889438841078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,256,0.009080000221729279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,128,0.008820444345474243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,64,0.0086986662613021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,32,0.009080000221729279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,16384,0.010337777435779572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,12288,0.006770666688680649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,65536,0.05403466688262092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,10240,0.006199999815887875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,512,512,0.009333333207501305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,8192,0.005519999812046687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,65536,0.08643733130560981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,16384,0.02702399922741784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,7168,0.006736889067623351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,12288,0.02251822253068288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,6144,0.00620177760720253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,10240,0.020252444677882724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,8192,0.017981333865059745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,5120,0.00573511090543535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,4096,0.005116444494989184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,7168,0.01700266698996226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,6144,0.01588000026014116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,3584,0.005051555732885997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,5120,0.014739554789331226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,2560,0.004679999831649992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,3072,0.004519999855094486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,4096,0.013430222041077085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,2048,0.003929777691761653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,3584,0.012934221989578672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,3072,0.012362666428089142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,2560,0.011808888779746162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,1536,0.004102222207519743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,1024,0.003605333467324575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,2048,0.01110311100880305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,768,0.004509333521127701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,512,0.003585777762863371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,256,0.0026977778308921387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,128,0.0021342221233579847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,1536,0.010560000108347999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,1024,0.009826666778988307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,768,0.009672889278994666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,64,0.0020862221717834473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,256,32,0.0020746667351987627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,512,0.009311999711725447
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,256,0.00902133352226681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,65536,0.015671999918089975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,16384,0.007098666495747036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,128,0.008842666943868002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,64,0.008784888519181145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,12288,0.00620177760720253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,10240,0.005740444279379315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,256,32,0.009038222332795462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,8192,0.005192888693677055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,16384,0.02697511182890998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,65536,0.08085333638721041
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,7168,0.0064444442590077715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,12288,0.02253066665596432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,10240,0.02028888960679372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,6144,0.006067555397748947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,5120,0.0047022220161226065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,8192,0.01795377830664317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,4096,0.004263111286693149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,3584,0.004497777670621872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,7168,0.01697688963678148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,6144,0.015764445066452026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,5120,0.014658666319317289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,3072,0.0041857775714662345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,2560,0.004450666821665234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,4096,0.013428444663683573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,2048,0.004023111114899318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,1536,0.003966222206751506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,3584,0.012878222597969903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,3072,0.012334222594896952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,1024,0.003485333174467087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,2560,0.011787555283970304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,768,0.00488800017370118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,2048,0.01112711098459032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,512,0.0037866665257347953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,256,0.0026702222724755607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,1536,0.010498666928874122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,128,0.002104000084929996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,1024,0.009881777895821465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,64,0.0021048889805873237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,768,0.009726222190592024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,128,32,0.0020951111283567217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,512,0.00924444446961085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,256,0.00907466643386417
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,128,0.008857777549160851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,16384,0.00685422205262714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,65536,0.010197333163685268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,64,0.008584889272848764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,12288,0.005983110931184556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,128,32,0.009141333401203156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,10240,0.005515555540720622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,8192,0.005013333426581489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,65536,0.08097955253389147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,16384,0.02699377801683214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,7168,0.004998222407367495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,12288,0.02238399949338701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,10240,0.02014844450685713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,6144,0.004586666822433472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,8192,0.017960889471901786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,5120,0.004509333521127701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,4096,0.0040755557517210645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,3584,0.004369777937730153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,7168,0.016991110311614144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,6144,0.015799999237060547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,3072,0.004060444318585926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,5120,0.01464000013139513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,2560,0.00434311106801033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,4096,0.013402666482660504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,3584,0.012907555533779992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,2048,0.003581333491537306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,1536,0.008225777910815345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,3072,0.01237066669596566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,1024,0.006026666611433029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,2560,0.01183644433816274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,768,0.004886222382386525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,2048,0.011016888750924004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,1536,0.010585777461528778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,512,0.00378577783703804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,1024,0.00979288915793101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,256,0.0026604444202449587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,128,0.0021057778762446507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,768,0.009683555199040307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,64,0.0021111110432280437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,512,0.0092257774538464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,64,32,0.0020684444655974707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,256,0.008991999758614434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,128,0.00871377769443724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,64,0.008810666700204214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,65536,0.009844444692134857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,16384,0.006763555523422029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,64,32,0.009011555876996782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,12288,0.0058346668051348785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,10240,0.005377777748637729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,65536,0.07659822040134005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,8192,0.004920000003443824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,16384,0.02583822276857164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,7168,0.004878222114510006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,10240,0.01945333348380195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,6144,0.004442666553788715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,12288,0.021572444174024794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,8192,0.0174026671383116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,5120,0.004373333520359463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,4096,0.003937777959638172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,7168,0.016404444972674053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,3584,0.017088888419999015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,3072,0.014766222900814481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,6144,0.015275556180212231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,5120,0.01422933406300015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,2560,0.012663111090660095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,4096,0.013183111117945777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,2048,0.010446222292052375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,1536,0.008251555263996124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,3584,0.012588444683286877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,1024,0.005978666659858491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,3072,0.012087110843923358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,2560,0.011566222541862063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,768,0.004804444395833545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,2048,0.010866666833559671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,512,0.0038035557501845886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,256,0.0026186667382717133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,1536,0.010382222632567087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,128,0.002124444478087955
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,1024,0.00980355590581894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,64,0.0020622221959961788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,768,0.009572444690598382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,16,32,32,0.0020764445265134177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,512,0.009215110705958473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,256,0.008940444224410588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,128,0.008738666772842407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,64,0.008780444661776224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,16,32,32,0.009158222211731805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,10240,1.1686124801635742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,12288,1.4063520431518555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,8192,1.5230426788330078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,16384,1.8447439405653212
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,10240,1.8548683590359156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,8192,0.900570657518175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,7168,0.7543706893920898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,12288,2.396518283420139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,6144,0.6254506640964085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,7168,1.301571528116862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,6144,1.1298924552069771
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,5120,0.9317644966973199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,16384,2.9631936815049915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,5120,0.5560631222195095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,4096,0.4419253402286106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,4096,0.7469226519266764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,3584,0.6551182005140517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,3072,0.31264977984958225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,2560,0.26643911997477215
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,3072,0.5626373291015625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,3584,0.3947742250230577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,2560,0.46998755137125653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,2048,0.40878934330410427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,1024,0.1920177804099189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,1024,0.12063911226060654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,768,0.15424356195661756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,2048,0.23268000284830728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,512,0.10489422082901001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,1536,0.1811466614405314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,128,0.013574221895800697
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,64,0.004972444640265571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,256,0.05226933293872409
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,768,0.09955555862850612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,512,0.08369155724843343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,32,0.004046222401989831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,256,0.07899910873836942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,64,0.08209155665503608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,65536,1536,0.3083244429694282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,128,0.08333510822719997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,65536,32,0.08230666981803046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,12288,0.36792887581719297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,16384,0.47781954871283633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,10240,0.514153798421224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,12288,0.6770479944017199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,16384,0.8225982454088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,10240,0.30523644553290474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,8192,0.22859999868604872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,8192,0.38420711623297793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,7168,0.22009778022766113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,7168,0.36099288198683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,5120,0.2352142333984375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,65536,1.6291493309868708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,6144,0.19159911738501656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,4096,0.188972446653578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,5120,0.1568346685833401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,6144,0.3172417746649848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,4096,0.11822133594089085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,3072,0.14224444495307073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,3584,0.16575911309983996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,3584,0.10652710994084676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,2560,0.13090844949086508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,3072,0.10109244452582465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,1536,0.07139822509553698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,1024,0.053505778312683105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,1536,0.052419556511773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,768,0.04085777865515815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,2560,0.08491110801696777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,512,0.008588444855478074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,2048,0.06765511300828722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,2048,0.1042871144082811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,1024,0.03396177623007033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,128,0.003563555578390757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,512,0.012315555579132505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,256,0.004648000001907349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,256,0.01130222198035982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,128,0.010688888529936472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,768,0.01351288871632682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,64,0.002938666691382726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,32,0.010043555663691627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,16384,64,0.010749333434634738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,32,0.00240622212489446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,16384,0.32901332113477916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,12288,0.47646135754055446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,12288,0.2781955666012234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,16384,0.5945342381795248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,16384,65536,3.222498575846354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,10240,0.37813509835137266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,10240,0.20973422792222765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,8192,0.17246577474806044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,7168,0.2480622132619222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,7168,0.14792088667551676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,8192,0.3162906699710422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,6144,0.12606310844421387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,6144,0.21899021996392143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,5120,0.1776862276924981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,4096,0.15333155790964761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,4096,0.09086844656202529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,3584,0.07996444569693671
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,3072,0.11462044715881348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,65536,1.2294240527682834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,3584,0.13710489537980822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,2560,0.09535555707083808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,5120,0.1175502273771498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,1536,0.057935112052493624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,1024,0.04271466533342997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,768,0.03285155693689982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,2048,0.08294577731026544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,1024,0.013035555680592855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,1536,0.040714667903052434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,3072,0.07456888755162557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,512,0.005623111294375525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,2560,0.0636062224706014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,2048,0.051783998807271324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,256,0.004077333129114575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,128,0.003237333355678452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,32,0.0022675556441148124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,64,0.002752888947725296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,768,0.012262221839692859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,256,0.010873777998818291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,64,0.010185778141021729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,128,0.010183999935785929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,512,0.011330666641394297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,12288,32,0.009493333597977957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,12288,65536,2.2119127909342446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,12288,0.2139351103040907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,16384,0.2780115604400635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,12288,0.3905484411451552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,10240,0.29351732465955943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,8192,0.24159023496839735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,8192,0.1407111088434855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,16384,0.5140470928615993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,7168,0.12340622478061253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,7168,0.20653333928849962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,10240,0.19303645028008354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,6144,0.18277155028449166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,6144,0.10705688926908706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,4096,0.11801244152916802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,5120,0.16285778416527644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,4096,0.08110755681991577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,65536,1.0244649251302083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,3072,0.08920088741514419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,5120,0.09878489043977524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,2560,0.08363378047943115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,3072,0.06300266583760579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,3584,0.06948444578382704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,3584,0.11483466625213623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,2048,0.060306668281555176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,1024,0.034875555170906916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,1536,0.020069332586394418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,2048,0.04429599973890516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,768,0.006911110960774952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,1024,0.01219288922018475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,1536,0.05063822203212314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,512,0.005543111099137201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,2560,0.05318577753172981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,768,0.011610666910807291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,128,0.0030826667530669104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,64,0.0028231110837724474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,256,0.003847111016511917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,512,0.011172444456153445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,128,0.009974222216341231
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,32,0.0022337778160969415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,32,0.009369778136412302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,256,0.010605333579911126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,10240,64,0.010103999740547603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,10240,65536,1.8443582322862413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,16384,0.3746115631527371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,16384,0.22089600563049316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,10240,0.23460798793368867
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,12288,0.18300088246663412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,12288,0.34728267457750106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,8192,0.19554221630096436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,10240,0.14899110794067383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,65536,0.7906017833285861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,6144,0.14647911654578313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,7168,0.1852017773522271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,7168,0.10761688815222846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,8192,0.12078221638997395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,6144,0.0875813364982605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,5120,0.07504977782567342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,3584,0.09364177783330281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,5120,0.13276533285776773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,4096,0.09525155358844334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,4096,0.06575644678539701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,3584,0.05466399921311272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,2048,0.054591112666659884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,2560,0.06764266888300578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,1536,0.04146044452985128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,3072,0.05014400018586052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,65536,1.4770666758219402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,3072,0.08052533202701144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,768,0.0063324446479479475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,1536,0.013471110827393003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,1024,0.012364444633324942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,512,0.004854222138722737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,2560,0.04332177837689718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,1024,0.01222400036123064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,768,0.011872000164455838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,256,0.0036640001667870414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,128,0.002965333354141977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,64,0.0027831111931138565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,8192,32,0.0023102222217453853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,512,0.01125511113140318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,2048,0.030400888787375555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,64,0.010560000108347999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,128,0.010281778044170803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,256,0.010693333215183683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,8192,32,0.009911999934249455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,16384,0.18996711572011313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,10240,0.20568622483147514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,12288,0.15342666043175593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,10240,0.12828532854715982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,16384,0.3702977763281928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,12288,0.30761154492696124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,8192,0.17198488447401258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,8192,0.09834933280944824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,6144,0.12811377313401964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,7168,0.1623928944269816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,5120,0.10358399815029568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,6144,0.08023377921846178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,65536,0.6849102444118924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,7168,0.0930746661292182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,3072,0.06328177452087402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,3584,0.07320444451438056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,3584,0.05024622215165032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,4096,0.08381778001785278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,5120,0.06848355796602038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,2560,0.05312800076272753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,3072,0.044063998593224414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,1536,0.032716443141301475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,4096,0.05630044142405192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,1024,0.007356444166766272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,2048,0.01534844438234965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,2048,0.0481279989083608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,768,0.006199999815887875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,512,0.004688888788223267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,1024,0.01181777815024058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,256,0.003608889049953885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,768,0.011477332976129321
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,128,0.0028924445311228433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,65536,1.292085329691569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,1536,0.012800888882742988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,512,0.011057777537239922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,64,0.0026657777941889236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,2560,0.03692977627118429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,256,0.010281778044170803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,7168,32,0.0026702222724755607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,128,0.010053333308961656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,64,0.010428444378905825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,7168,32,0.009701333112186855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,10240,0.17668622069888643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,16384,0.2860017882453071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,12288,0.12048266993628608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,16384,0.17860621876186797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,12288,0.23611201180352104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,8192,0.14819910791185167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,10240,0.10299733612272476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,8192,0.08584977520836724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,7168,0.12406222025553386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,7168,0.07560888926188152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,65536,0.5902755525377061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,6144,0.11024177736706203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,5120,0.08926488955815633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,6144,0.06566755639182197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,5120,0.05588622225655449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,4096,0.07257599963082208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,4096,0.0450160006682078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,3584,0.04068266683154636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,3072,0.05757688813739353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,2560,0.05128622386190626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,3584,0.06649777624342176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,2048,0.03730400072203742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,1536,0.028393778536054824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,65536,1.1076506508721244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,2048,0.01296622223324246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,3072,0.03588533401489258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,1024,0.006892444358931647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,768,0.005702222386995952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,2560,0.02200711104604933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,1536,0.012084444363911947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,512,0.004383111165629493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,128,0.0028195555011431375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,1024,0.011298666397730509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,768,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,64,0.0026693333768182327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,512,0.010655110908879174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,256,0.010168000227875179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,32,0.0025928889711697898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,128,0.009890666438473595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,6144,256,0.0034728890491856467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,32,0.009296889106432596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,6144,64,0.010088000032636855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,12288,0.10241333643595378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,16384,0.13699644141727024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,10240,0.1480648914972941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,16384,0.24018488989935982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,12288,0.22223822275797525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,7168,0.10437244176864624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,8192,0.12450844711727566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,10240,0.08726755777994792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,8192,0.07261955738067627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,5120,0.07490933603710599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,65536,0.4951804478963216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,6144,0.09264710876676772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,7168,0.06994489166471693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,4096,0.06076088878843519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,3584,0.05316533313857185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,5120,0.04663199848598904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,3072,0.04622133241759407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,65536,0.9245093663533529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,2560,0.038726223839653864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,3072,0.02011022302839491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,2560,0.012972444295883179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,3584,0.0373306671778361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,2048,0.0315048893292745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,4096,0.042394667863845825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,2048,0.012320888539155325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,1024,0.00665244460105896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,768,0.005527110977305307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,1536,0.01166488892502255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,512,0.004397333496146732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,256,0.003419555516706573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,1536,0.011035555766688453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,128,0.0028328889360030494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,512,0.010519111322032081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,6144,0.06030133035447863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,256,0.009840889109505547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,64,0.00254133323000537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,5120,32,0.002513777878549364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,1024,0.011062222222487131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,128,0.009720000127951304
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,64,0.010024888647927178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,32,0.009367110828558603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,5120,768,0.010834666589895884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,16384,0.1939173274570041
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,12288,0.1590577761332194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,12288,0.08482311169306438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,10240,0.13336443901062012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,16384,0.1206924385494656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,10240,0.07129244671927558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,8192,0.1005013320181105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,7168,0.08429688877529568
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,8192,0.05997422006395128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,7168,0.05667110946443346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,65536,0.3998097843594021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,6144,0.04493688874774509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,4096,0.05507555603981018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,5120,0.06050578090879652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,6144,0.08346488740709092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,5120,0.03884889019860162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,65536,0.7417822413974338
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,3072,0.03767377800411648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,3584,0.014720888601409065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,3584,0.04806488752365112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,3072,0.012856889102194043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,2048,0.012463111016485425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,2560,0.012153777811262341
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,1536,0.008343111309740279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,1024,0.006193777753247156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,2560,0.03152444296412998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,768,0.005207111024194294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,512,0.004233777936961916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,1536,0.01127644462717904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,2048,0.011810666984981961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,256,0.0033235556135574975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,4096,0.025921778546439275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,512,0.010140444669458602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,1024,0.010371555884679159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,128,0.002777777819169892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,64,0.002640888922744327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,768,0.010252444280518426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,4096,32,0.002463999928699599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,256,0.009570666485362584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,64,0.009510222408506606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,128,0.009214222431182861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,4096,32,0.008760888543393876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,16384,0.17040622234344482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,12288,0.13935200373331705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,16384,0.0995840032895406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,12288,0.07538755734761556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,10240,0.11721866660647923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,8192,0.08821333116955227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,65536,0.35327733887566465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,10240,0.06369422541724311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,7168,0.07433155510160658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,7168,0.04577955603599548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,5120,0.05331200030114916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,4096,0.04385155439376831
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,6144,0.06719111071692573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,6144,0.0402453343073527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,3584,0.038176000118255615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,4096,0.013778666655222574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,3584,0.013010666602187686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,2560,0.024497777223587036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,5120,0.033420443534851074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,65536,0.6494186719258627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,3072,0.01236355553070704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,2048,0.01014666673209932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,3072,0.03735911183887058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,8192,0.052186667919158936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,1536,0.008291555775536431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,1024,0.0061048890153567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,768,0.005129777722888523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,2560,0.011968000067604912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,512,0.004201777693298128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,2048,0.011425777441925473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,1536,0.010820444259378644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,1024,0.01037333326207267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,256,0.003289777785539627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,128,0.0027386666172080566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,512,0.00981244444847107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,64,0.002434666578968366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,256,0.009402666654851701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3584,32,0.002416888872782389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,128,0.008925333619117737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,64,0.009309333231714036
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,768,0.010132444401582083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3584,32,0.008832889298597971
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,16384,0.08577422300974528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,10240,0.09081511365042792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,12288,0.12023466163211399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,16384,0.14720443884531656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,12288,0.07192533546023898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,8192,0.07636711332533094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,7168,0.06432622008853488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,10240,0.0549759997261895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,65536,0.30823998981051975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,5120,0.046852442953321666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,6144,0.05765066544214884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,8192,0.04479111234347025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,7168,0.04402133491304186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,5120,0.01737600068251292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,6144,0.03575199842453003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,65536,0.5578835805257162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,3584,0.03330488999684652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,4096,0.038273778226640486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,2560,0.014193778236707052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,2048,0.00996800015370051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,3584,0.012851555314328937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,1536,0.008211555580298105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,2560,0.01184266640080346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,3072,0.03271111183696323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,4096,0.013364444176355997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,2048,0.011266666981909009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,1024,0.005985777825117111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,768,0.005063999858167437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,1536,0.010775110787815519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,512,0.004117333226733738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,256,0.0032595555401510666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,1024,0.010263111028406356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,3072,0.012326222327020435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,768,0.009904888768990835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,128,0.002728888971938027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,512,0.009679999616410997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,64,0.0023635555472638872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,3072,32,0.0023626666516065598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,256,0.009272000028027428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,64,0.009154666629102495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,32,0.008642666869693333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,3072,128,0.00904088881280687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,16384,0.12352089087168376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,12288,0.10145866870880127
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,16384,0.0705911119778951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,10240,0.0779013302591112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,65536,0.25954845216539174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,12288,0.057068447271982826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,8192,0.06486577457851835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,7168,0.05465333329306709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,10240,0.04639022217856514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,8192,0.038819554779264666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,65536,0.4666124449835883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,6144,0.04956000049908956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,4096,0.03276266654332479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,5120,0.04074133435885111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,6144,0.01868799991077847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,3584,0.024180443750487432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,5120,0.014375110467274984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,3072,0.014074666632546319
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,2560,0.012101333174440594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,7168,0.03569866551293267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,3584,0.012777778009573618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,2048,0.009490666290124258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,1536,0.008032889001899296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,3072,0.012308444413873883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,1024,0.005821333163314396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,2560,0.011706666813956367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,768,0.00527555536892679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,2048,0.011194667054547204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,4096,0.013256000147925483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,1536,0.010686222049925061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,512,0.004173333446184794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,128,0.0027137777457634607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,1024,0.010109333528412713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,768,0.009883555273214975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,64,0.002332444406217999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,512,0.009584888815879822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,32,0.002293333411216736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,128,0.008832000195980072
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,256,0.009176889227496253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,64,0.009160888691743216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2560,32,0.008439111212889353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2560,256,0.0032266666077905228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,12288,0.08372800217734443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,16384,0.10400444269180298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,16384,0.05831378036075168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,12288,0.04682666725582547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,65536,0.21308533350626627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,8192,0.05461688836415609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,10240,0.07360000080532499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,65536,0.37465421358744305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,7168,0.046824001603656344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,10240,0.039015998442967735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,6144,0.041217777464124895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,5120,0.03496266735924615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,4096,0.022209778428077698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,8192,0.023715555667877197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,6144,0.015281778242852954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,3584,0.015097777048746744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,5120,0.014223999447292753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,7168,0.016629333297411602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,3072,0.01330488920211792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,3584,0.012622222304344177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,2048,0.009344889058007134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,1536,0.007835555407736037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,2560,0.011623111036088733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,2560,0.011522666447692446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,3072,0.012152888708644442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,1024,0.005658666706747479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,2048,0.011018666956159802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,1536,0.010568000376224518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,512,0.003936888857020272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,4096,0.013076444466908773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,768,0.004923555586073133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,128,0.00263288886182838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,256,0.003140444349911478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,1024,0.009899555808968013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,768,0.00978311151266098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,64,0.002283555558986134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,256,0.009146666361225976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,2048,32,0.002266666748457485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,128,0.008877333667543199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,512,0.009488888912730748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,64,0.009165333376990424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,2048,32,0.00853066643079122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,16384,0.07694133122762044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,12288,0.05752711163626777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,16384,0.04743644595146179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,8192,0.047000000874201454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,12288,0.03789155681927999
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,65536,0.16479378276401097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,10240,0.02045866681469811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,10240,0.052616887622409396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,65536,0.28306489520602757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,7168,0.041915555795033775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,6144,0.03605155481232537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,7168,0.016216889023780823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,5120,0.027207111318906147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,8192,0.017089777522616915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,6144,0.015190222197108798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,4096,0.016401777664820354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,4096,0.012977778083748288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,3584,0.014869333969222175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,3072,0.01296533313062456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,2560,0.011175110936164856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,1536,0.0075457774930530125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,2048,0.009212444225947062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,3072,0.012120889292822944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,1024,0.005579555614127054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,5120,0.014279999666743808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,2048,0.010926222635640038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,3584,0.01258488910065757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,768,0.0047493332790003884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,2560,0.011537777880827585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,1536,0.01052177780204349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,512,0.0038035557501845886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,1024,0.009943111075295342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,256,0.00305244450767835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,128,0.002426666725012991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,768,0.009703999592198266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,64,0.0023928888969951207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,512,0.009370666411187913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1536,32,0.0021973333011070886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,256,0.009081777599122789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,128,0.0088764445649253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,64,0.009191110730171204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1536,32,0.008513777620262569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,12288,0.038190222448772855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,16384,0.05187111099561056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,16384,0.029346668057971533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,65536,0.19152532683478463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,10240,0.03278399838341607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,12288,0.021312889125612047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,65536,0.11773867077297634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,10240,0.01921688848071628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,6144,0.007631999750932057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,7168,0.008397333323955536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,8192,0.017086222767829895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,5120,0.006992888947327931
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,4096,0.006140444427728653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,7168,0.0162124451663759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,3584,0.006135111053784688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,8192,0.010838222172525195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,5120,0.014210666219393412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,3072,0.005488889084921942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,2560,0.010956444674068026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,4096,0.01293333371480306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,3584,0.01260355528857973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,6144,0.015055111712879606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,1536,0.007391110890441471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,3072,0.012033777932325998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,1024,0.005459555735190709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,2048,0.009168888959619734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,2560,0.011565333439244164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,768,0.004612444589535396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,2048,0.010879110958841113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,512,0.0037288889288902283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,256,0.0028533333291610083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,1536,0.01053155544731352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,128,0.002279111080699497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,1024,0.009849777652157677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,768,0.00960799968904919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,64,0.0022906667242447534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,512,0.009373333719041612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,1024,32,0.002157333410448498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,256,0.00905688852071762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,128,0.008812444077597724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,16384,0.03959555427233378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,64,0.009191110730171204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,12288,0.029520889123280842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,1024,32,0.008397333323955536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,65536,0.14938933319515652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,10240,0.009759110709031424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,8192,0.0084915558497111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,65536,0.10297866662343343
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,16384,0.0251884443892373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,12288,0.021186666356192693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,7168,0.008254222571849823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,10240,0.019167999426523846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,6144,0.007496000164084964
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,5120,0.007169778148333232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,4096,0.00627377794848548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,8192,0.017062221964200337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,7168,0.016065778003798593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,3584,0.005616888817813661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,6144,0.015055111712879606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,5120,0.014055111342006259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,3072,0.005108444641033809
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,2560,0.0052488889131281115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,4096,0.012891555825869242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,2048,0.004589333302444882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,3584,0.012491555677519904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,3072,0.01201333353916804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,1536,0.007282666862010956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,1024,0.005408000200986862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,768,0.004503111044565837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,2560,0.01145066652033064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,2048,0.010819555156760745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,1536,0.010311110979980892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,512,0.0035982223020659555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,256,0.0027822222974565295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,1024,0.00977955593003167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,128,0.002187555655837059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,64,0.002144888871245914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,512,0.009302222066455418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,768,32,0.0021199999998013177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,768,0.009672889278994666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,256,0.009013333254390294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,128,0.008861333131790161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,16384,0.014312000738249885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,64,0.009119110802809397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,768,32,0.00853066643079122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,12288,0.009272889130645329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,10240,0.008194666769769456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,65536,0.10118666622373794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,8192,0.007407111426194509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,16384,0.02512977851761712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,12288,0.021242666575643752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,7168,0.0069191112286514705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,10240,0.01926666663752662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,65536,0.09129955371220906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,6144,0.006434666613737742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,8192,0.0170151111152437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,5120,0.005932444499598608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,3584,0.005207111024194294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,4096,0.005234666582610872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,7168,0.01609066625436147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,6144,0.015090665883488126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,3072,0.004769777672158347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,5120,0.014074666632546319
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,4096,0.012880889077981314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,2560,0.004869333157936732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,2048,0.004302222281694412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,3584,0.012472000386979846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,1536,0.004281777888536453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,1024,0.00535022219022115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,2560,0.011502222054534487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,768,0.00444177786509196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,2048,0.010797333386209277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,3072,0.011940444509188334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,1536,0.010384000009960597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,512,0.003536000019974179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,256,0.0027155555370781156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,128,0.002128888956374592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,1024,0.009723555710580613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,64,0.0020755556308560902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,768,0.0096142225795322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,512,32,0.0020871110674407748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,512,0.009232000344329411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,256,0.008935111264387766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,128,0.008808000220192803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,65536,0.05364177624384562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,16384,0.010087110930018956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,64,0.009127111070685917
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,12288,0.008397333323955536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,512,32,0.008479110896587372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,10240,0.005852444304360284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,8192,0.005221333354711533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,7168,0.0064311110311084324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,16384,0.025115556187099878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,12288,0.021174222230911255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,65536,0.08044888575871785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,6144,0.005993777679072485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,10240,0.019140443868107267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,8192,0.01703733371363746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,5120,0.005499555418888728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,4096,0.0048942222363419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,3584,0.004917333523432414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,7168,0.016131555040677387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,6144,0.01498133275243971
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,3072,0.004264889078007804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,2560,0.004491555607981152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,5120,0.014075555735164218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,4096,0.012886222038004132
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,2048,0.004003555410438114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,3584,0.012441777520709567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,3072,0.011889778077602386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,1024,0.003528888854715559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,1536,0.004021333323584663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,2560,0.011396444506115384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,768,0.0047679998808436925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,2048,0.010819555156760745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,512,0.0037120001183615792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,1536,0.010399999717871347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,256,0.002675555646419525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,1024,0.009749333063761393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,128,0.0021057778762446507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,64,0.0020862221717834473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,768,0.00962222201956643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,512,0.009205333060688442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,256,32,0.00208177769349681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,128,0.008793777889675563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,65536,0.015662221444977652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,16384,0.006821333534187741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,32,0.008433777425024245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,64,0.009171555439631144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,256,256,0.008954666554927826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,12288,0.005921777751710679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,10240,0.005491555564933353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,8192,0.004978666702906291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,65536,0.07382844554053412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,16384,0.025073778298166063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,12288,0.02113688819938236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,10240,0.01917866700225406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,7168,0.004935111022657818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,6144,0.004495111190610462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,5120,0.004516444272465176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,8192,0.01703377730316586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,7168,0.016028443972269695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,4096,0.004761777818202972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,3584,0.0047768888374169665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,6144,0.015026667051845126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,3072,0.00434044458799892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,5120,0.014081777797804939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,2560,0.004351110921965705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,4096,0.012821333275900947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,2048,0.0038613333470291565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,3584,0.012435555458068848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,3072,0.011870221959220039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,1536,0.007961778177155389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,2560,0.011423110961914062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,1024,0.005838222387764189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,2048,0.010786666638321347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,768,0.0047626665068997275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,512,0.0036924444139003754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,256,0.0026346666531430352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,1536,0.010290666586822933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,1024,0.009773333039548662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,128,0.0020879999630981022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,768,0.00962400022480223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,64,0.0020844443804687923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,512,0.009303111169073317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,128,32,0.0020675555699401433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,256,0.008936000367005667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,65536,0.009703999592198266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,16384,0.006636444479227066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,128,0.008595555192894405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,64,0.009193778038024902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,12288,0.005730666634109285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,128,32,0.008467555873923832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,10240,0.005280888742870755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,8192,0.004829333474238713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,65536,0.07390666670269437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,7168,0.004811555561092165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,16384,0.025078222155570984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,12288,0.021137777302000258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,10240,0.019144000278578866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,6144,0.004400889078776042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,5120,0.004332444320122401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,8192,0.016977777083714802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,4096,0.003933333274390963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,7168,0.016065778003798593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,3584,0.016393777396943834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,3072,0.014303111367755465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,6144,0.014931556251313953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,5120,0.014010666145218743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,2560,0.012198222180207571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,4096,0.012889777620633444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,2048,0.010059555371602377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,3584,0.012413333687517377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,1536,0.00792266676823298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,3072,0.011868444581826528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,1024,0.0058293334311909145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,2560,0.011412444214026133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,2048,0.010749333434634738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,768,0.0047395556337303585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,512,0.00370755543311437
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,1536,0.010317333042621613
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,256,0.0026311110705137253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,128,0.0020951111283567217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,1024,0.009709333380063375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,768,0.009409777820110321
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,64,0.0020639999873108333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,64,32,0.0020711111525694528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,512,0.009179555707507664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,256,0.00886399961180157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,128,0.008767111433876885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,16384,0.006543111056089401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,65536,0.009299555586444007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,64,0.008969777988062965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,64,32,0.008375110725561777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,12288,0.005664000080691443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,10240,0.005214222189452913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,8192,0.004746666798988978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,16384,0.024950222836600408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,65536,0.07327644692526923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,12288,0.020976000361972388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,10240,0.018933332628673978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,7168,0.031159109539455835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,6144,0.02698844505680932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,5120,0.02252799934811062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,8192,0.016898666818936665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,4096,0.018431110514534842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,7168,0.015879111157523263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,3584,0.016368000043763053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,6144,0.014853333433469137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,3072,0.014108444253603617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,5120,0.013978666729397245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,2560,0.012141333685980903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,4096,0.012796444197495779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,2048,0.010092444717884064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,3584,0.012312889099121094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,1536,0.007840889195601145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,3072,0.01184800018866857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,1024,0.005809777726729711
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,2560,0.01126933346192042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,768,0.0047315553658538395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,512,0.003630222131808599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,2048,0.01074577785200543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,1536,0.010240889257854885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,1024,0.009681777821646797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,256,0.0025822222232818604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,128,0.002089777754412757
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,768,0.009480888644854227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,64,0.0020497778637541667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,8,32,32,0.002058666613366869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,512,0.009233777721722921
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,256,0.008963555925422246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,128,0.008803555534945594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,64,0.009024000002278222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,8,32,32,0.008417777717113495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,10240,1.1610177357991536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,12288,1.4019599490695531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,8192,1.5154711405436199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,16384,1.8407412634955511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,10240,1.848489761352539
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,8192,0.8944764667087131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,12288,2.3874009450276694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,7168,0.7532115512424045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,6144,0.6853840086195203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,7168,1.2954400380452473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,5120,0.5090159840053982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,16384,2.9545849694146047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,6144,1.1263484954833984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,5120,0.9270835452609592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,4096,0.7430773311191134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,4096,0.440337790383233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,3584,0.3567955493927002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,3072,0.3089795642428928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,2560,0.2628755569458008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,1536,0.2819235589769152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,3072,0.6085857815212673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,2048,0.23386043972439238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,3584,0.7105920049879285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,2560,0.5101031197441949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,2048,0.4067066775427924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,768,0.15266756216684976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,512,0.10230222013261582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,1536,0.17953956127166748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,128,0.007686222592989604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,256,0.052208890517552696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,768,0.10130755768881904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,1024,0.20650577545166016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,512,0.08304533031251696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,256,0.07902222209506564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,128,0.0819715526368883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,64,0.07866755459043714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,1024,0.12792355484432644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,32,0.004055110944641961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,65536,64,0.00489155575633049
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,65536,32,0.08146578073501587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,10240,0.297614230049981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,12288,0.3682933383517795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,16384,0.47305864757961696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,10240,0.5123128890991211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,12288,0.6757386525472006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,16384,0.8205617798699273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,8192,0.2270053227742513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,7168,0.21974044375949434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,7168,0.3595946629842122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,8192,0.4141351117028131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,6144,0.3144177860683865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,6144,0.19066045019361708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,5120,0.23698933919270834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,4096,0.12149511443244086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,4096,0.20650933848487008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,5120,0.15843466917673746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,3072,0.1418248944812351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,3584,0.11715643935733372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,3072,0.09680977794859146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,2048,0.09471378061506484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,2560,0.1304524474673801
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,2560,0.0832808878686693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,1024,0.048362665706210665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,3584,0.1809013287226359
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,65536,1.6534408993191188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,2048,0.06719911098480225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,512,0.012645333177513547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,1536,0.07874755726920234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,768,0.04079644547568427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,768,0.013072888884279462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,1024,0.03004266818364461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,128,0.003463111196955045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,256,0.004478222380081813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,512,0.012443555725945367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,128,0.011217777927716574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,256,0.01129066695769628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,1536,0.052221334642834134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,64,0.009999111294746399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,64,0.0026808888134029177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,32,0.0023688889212078517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,16384,32,0.009646221995353699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,12288,0.2528497907850477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,16384,0.35941510730319554
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,16384,65536,2.947681850857205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,12288,0.5059608883327907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,16384,0.5871786541408962
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,10240,0.3562053309546576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,10240,0.2289795610639784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,7168,0.26131200790405273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,8192,0.17300266689724395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,7168,0.14741510815090603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,8192,0.3284951051076253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,6144,0.12728622224595812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,65536,1.2222977744208443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,6144,0.24541687965393066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,5120,0.11854578389061822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,4096,0.15618933571709528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,3584,0.13243999746110705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,5120,0.20601244767506918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,3584,0.08060533470577665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,4096,0.0988888872994317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,2048,0.0763253304693434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,3072,0.12372088432312012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,2560,0.058127111858791776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,3072,0.0739955570962694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,2560,0.1048097809155782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,1024,0.03929422299067179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,2048,0.04699377881156074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,1536,0.03704977697796292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,768,0.03140177660518222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,512,0.005600888695981767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,256,0.004009777886999978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,128,0.0029493332323100832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,1536,0.06283111042446561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,768,0.011968889170222811
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,1024,0.012803555362754397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,32,0.002704888789190186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,256,0.010641777680979835
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,65536,2.212003495958116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,64,0.009512888888518015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,12288,64,0.002728888971938027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,512,0.011503111157152386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,128,0.010495111346244812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,12288,32,0.009113777842786577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,12288,0.21180889341566297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,10240,0.19180444876352945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,16384,0.2756631109449598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,10240,0.2919520007239448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,12288,0.38907644483778214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,8192,0.24094578954908583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,7168,0.12265333864423965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,16384,0.5098257594638401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,8192,0.15277243985070124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,6144,0.18140444490644667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,7168,0.20516444577111137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,5120,0.14739910761515299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,3584,0.10327377584245469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,5120,0.09920622242821588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,4096,0.12977422608269587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,6144,0.11665956179300944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,3072,0.08906933334138657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,4096,0.0813537769847446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,3072,0.06261599726147123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,2560,0.08323911163542005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,2560,0.04889333248138428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,2048,0.060157332155439586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,1024,0.03119022316402859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,65536,1.0186871422661674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,1536,0.05060355530844795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,3584,0.07146844598982069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,512,0.005623111294375525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,768,0.010117332968446944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,2048,0.04025955663786994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,768,0.01163644426398807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,256,0.0038053331275780997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,128,0.002784888984428512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,512,0.011117333339320289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,1536,0.021542222963439092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,32,0.0026364444444576898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,64,0.002630222174856398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,256,0.010328888893127441
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,128,0.010305778019958073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,1024,0.012167111039161682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,32,0.008950221869680617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,10240,64,0.009332444104883406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,10240,65536,1.8434452480740016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,16384,0.2173466682434082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,16384,0.374143123626709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,12288,0.1659919950697157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,10240,0.2339235676659478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,12288,0.346560001373291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,10240,0.13649688826666936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,8192,0.19493867291344535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,8192,0.11362044016520183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,65536,0.785737779405382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,6144,0.0896400014559428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,6144,0.1461928950415717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,7168,0.10881600115034316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,5120,0.11790400081210667
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,7168,0.1849022176530626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,3584,0.08315466509924994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,5120,0.07500800159242418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,4096,0.06057422028647529
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,3584,0.05364533265431722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,2560,0.06014844444062975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,3072,0.08069511254628499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,3072,0.04654755526118808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,4096,0.10696533653471206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,2048,0.04861244559288025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,1024,0.009340444372759925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,1536,0.041391998529434204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,65536,1.4763555526733398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,768,0.006021333237489064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,2048,0.030026667647891577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,256,0.00361333332127995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,768,0.011777777638700275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,512,0.011351111034552256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,1536,0.01298577752378252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,1024,0.012178666889667511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,2560,0.0428746673795912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,256,0.010674667027261523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,32,0.0026648888985315957
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,64,0.0026986667265494666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,512,0.004831111265553368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,64,0.009912889036867354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,8192,128,0.002748444469438659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,32,0.009544000029563904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,8192,128,0.0107360002067354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,16384,0.18776799572838676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,12288,0.1388853391011556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,12288,0.30747466617160374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,16384,0.36977778540717227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,10240,0.23048888312445748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,8192,0.17046666145324707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,10240,0.1175617774327596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,65536,0.6782364315456815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,8192,0.1045831110742357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,6144,0.12782222694820827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,7168,0.16228532791137695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,6144,0.07566310962041219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,7168,0.09357600079642402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,5120,0.11651821931203206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,3584,0.082296888033549
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,3072,0.06308622492684259
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,5120,0.06839111116197374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,2560,0.03492355677816603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,4096,0.09426222244898479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,2048,0.048194666703542076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,3072,0.04384533233112759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,4096,0.055771556165483266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,2560,0.059374219841427274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,2048,0.015607110328144498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,1536,0.03633066680696275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,1536,0.012472000386979846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,1024,0.011605333122942181
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,768,0.00600355573826366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,512,0.004726222405831019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,65536,1.291593763563368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,128,0.0026888888743188647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,1024,0.007215110792054071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,768,0.011347555451922946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,3584,0.04987644486957126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,32,0.0026480000880029467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,512,0.01090755561987559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,64,0.0026551110463009942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,128,0.010423111418883005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,256,0.010318222145239512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,32,0.009131555755933126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,7168,256,0.0035546666218174826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,7168,64,0.009676444033781687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,16384,0.28292621506585014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,12288,0.12878933217790392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,12288,0.23592533005608451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,16384,0.1774053308698866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,10240,0.19818133778042266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,10240,0.11069244146347046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,8192,0.14762843979729548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,7168,0.12440178129408096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,7168,0.08018577761120267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,8192,0.08476266596052383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,65536,0.5851235389709473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,6144,0.12399555577172174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,6144,0.06553333335452609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,5120,0.08919466866387261
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,5120,0.05578311284383138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,4096,0.08142755428949992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,3584,0.06325422392951117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,3584,0.040227555566363864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,2560,0.05123910970158047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,2048,0.03717155588997735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,4096,0.04857866631613838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,65536,1.1070435841878254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,3072,0.061321775118509926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,3072,0.0350542234049903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,2048,0.012637333737479316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,1536,0.028400000598695543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,768,0.005570666657553778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,1536,0.011783999701340994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,2560,0.019952888290087383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,512,0.004553777890072929
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,1024,0.011286222272449069
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,128,0.0028213332924577924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,256,0.003444444388151169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,768,0.01087288889620039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,1024,0.00685777763525645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,512,0.01054133309258355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,64,0.0025617778301239014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,256,0.010127999716334874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,6144,32,0.002562666725781229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,64,0.009377777576446533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,128,0.010178666975763109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,6144,32,0.008984000318580205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,16384,0.23965777291191948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,12288,0.19731556044684517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,12288,0.10198311010996501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,16384,0.13397688335842556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,10240,0.16575645075903997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,8192,0.12432800398932563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,7168,0.1042977770169576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,8192,0.07027910815344916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,7168,0.06800355513890584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,10240,0.08667111396789551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,6144,0.09210844172371759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,65536,0.49005868699815536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,4096,0.061230222384134926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,5120,0.07503288984298706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,3584,0.05327111151483324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,5120,0.04646311203638712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,6144,0.05539022220505608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,3584,0.03478755553563436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,2560,0.03871644536654154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,65536,0.9238470925225152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,2048,0.031488888793521456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,3072,0.019897777173254225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,4096,0.039429333474900984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,2560,0.012848888834317526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,2048,0.012048000262843238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,768,0.0053839998112784485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,1536,0.01143555591503779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,1024,0.01094133324093289
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,1536,0.009846222069528367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,768,0.0107022225856781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,512,0.004303110970391167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,256,0.0033653332955307434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,128,0.0028035555862718155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,1024,0.006543111056089401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,512,0.010288889209429422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,64,0.002521777732504739
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,128,0.009993777506881291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,32,0.002552888873550627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,256,0.009867555565304225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,64,0.009288888838556077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,5120,32,0.008817777865462834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,5120,3072,0.051855110459857516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,12288,0.15880000591278076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,16384,0.19341422451866996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,12288,0.08406044377221002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,16384,0.11787554952833389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,10240,0.06990222136179607
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,10240,0.13343200418684217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,8192,0.056002669864230685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,8192,0.11064622137281631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,65536,0.39502843221028644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,7168,0.09451466798782349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,7168,0.05060977737108866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,6144,0.08379377921422322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,6144,0.044455111026763916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,5120,0.06042222181955973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,4096,0.049230221245023936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,65536,0.7408302095201281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,4096,0.02477333280775282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,3584,0.04308266772164238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,2560,0.03144444359673394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,3072,0.037504888243145414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,3584,0.013609777722093793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,3072,0.012648888760142855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,1536,0.00812711069981257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,1024,0.006088888893524806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,5120,0.04227822356753879
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,2560,0.012028444144460889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,1536,0.010924444430404238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,512,0.0041911109454101985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,2048,0.011502222054534487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,768,0.010390222072601318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,256,0.0032915555768542816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,768,0.00510044437315729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,2048,0.013399110900031196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,128,0.0027831111931138565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,32,0.0024622221373849446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,256,0.009313777916961247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,1024,0.010501333408885531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,128,0.009595555563767752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,4096,64,0.002458666761716207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,64,0.00870488915178511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,512,0.00982488857375251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,4096,32,0.008454221818182204
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,16384,0.1697626643710666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,12288,0.13947378264533147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,16384,0.10396622286902533
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,12288,0.07459288835525513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,10240,0.10438399844699436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,8192,0.08744533194435967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,7168,0.0744417773352729
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,10240,0.06242844131257799
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,65536,0.3480222225189209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,8192,0.04990666773584154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,7168,0.04510133465131124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,5120,0.053411556614769824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,4096,0.04389511214362251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,6144,0.06627821922302246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,65536,0.6491004625956217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,4096,0.013487999637921652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,3072,0.03351022137535943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,3584,0.012825777961148156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,6144,0.039919111463758684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,5120,0.03680355681313409
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,2560,0.028002666102515325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,2048,0.010031110710567897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,1536,0.008090666598743862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,3584,0.03948622279696994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,3072,0.012317332956526013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,1024,0.005945777727497949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,2560,0.011865778101815118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,2048,0.011247999966144562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,768,0.005000888887378904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,512,0.0041288890772395665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,1536,0.010862222148312463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,256,0.003257777748836411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,128,0.0026728889594475427
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,1024,0.01035733355416192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,64,0.0024071110205517872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,256,0.009222221871217092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3584,32,0.0024133332901530797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,512,0.009734222458468543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,128,0.009545777406957414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,64,0.00869422240389718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,32,0.008110221889283922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3584,768,0.010049777726332346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,12288,0.12033243974049886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,16384,0.1470657719506158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,16384,0.08436889118618435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,12288,0.06659822331534491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,8192,0.07641866472032335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,10240,0.10163999928368463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,10240,0.054274668296178184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,8192,0.04425422350565592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,65536,0.3019911183251275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,7168,0.06441955433951484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,5120,0.04639822244644165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,6144,0.06437777810626559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,4096,0.03804711169666714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,65536,0.5569030973646376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,5120,0.018175111876593698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,6144,0.03458933366669549
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,2560,0.011967110964987012
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,3584,0.03348799877696567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,4096,0.013182222015327878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,7168,0.03991555505328708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,3584,0.012607999973826937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,1536,0.008014221986134848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,1024,0.0059164443777667145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,2560,0.011742221812407175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,3072,0.027445332871543035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,2048,0.011225777367750803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,768,0.004973333328962326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,512,0.004097777936193678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,2048,0.010013333625263637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,1024,0.01015822258260515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,1536,0.010563555690977307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,128,0.002420444455411699
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,768,0.009973333113723332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,64,0.0023644444429212147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,256,0.003212444484233856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,512,0.009694221946928237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,256,0.009167110754383935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,3072,32,0.0023591110689772502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,128,0.0092275556590822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,64,0.008547555241319869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,3072,0.012221333053376941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,3072,32,0.007977777885066139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,12288,0.101455999745263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,16384,0.07200533151626587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,16384,0.12462933858235677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,12288,0.05645510885450575
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,65536,0.2546604474385579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,10240,0.0870364440812005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,8192,0.06565244330300225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,10240,0.045913779073291354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,7168,0.054529776175816856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,8192,0.03841955463091532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,65536,0.4658168686760797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,6144,0.0491866668065389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,7168,0.03471466567781236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,6144,0.016338666280110676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,4096,0.03286222285694546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,5120,0.040170666244294904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,3584,0.02681422233581543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,4096,0.013047999805874295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,2560,0.011424000064531961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,5120,0.014127999544143677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,3584,0.012572444147533841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,1536,0.007689778175618913
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,3072,0.012132444315486483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,3072,0.017300445172521803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,2048,0.00961688905954361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,2048,0.011008000208271874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,768,0.005184000151024925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,1536,0.010614222122563256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,512,0.0038533334930737815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,1024,0.0057608890864584185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,1024,0.010033778018421596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,256,0.0031866667171319327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,2560,0.011633777783976661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,768,0.009866666462686326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,128,0.0026791110220882627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,64,0.0023288888235886893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,512,0.00962044464217292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2560,32,0.0023173333870040043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,256,0.008982222113344405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,128,0.009352888498041365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,64,0.008561777571837107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2560,32,0.00793599999613232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,16384,0.10314844714270698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,12288,0.08355110883712769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,16384,0.05995111332999336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,12288,0.046642667717403836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,10240,0.06678311030069987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,8192,0.054959111743503146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,10240,0.03869244456291199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,7168,0.04625688989957174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,65536,0.3740897708468967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,6144,0.041344000233544245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,8192,0.02233422299226125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,6144,0.015057777365048727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,7168,0.016232000456915963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,4096,0.023250665929582384
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,5120,0.034627556800842285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,3584,0.014775110615624322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,65536,0.20948621961805555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,4096,0.01290933373901579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,2560,0.011389333340856763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,3072,0.013015111287434896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,2048,0.009301332963837517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,3584,0.012455999851226807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,1536,0.0075626663035816616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,3072,0.011996444728639392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,5120,0.013995556367768182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,2048,0.01094044413831499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,2560,0.01145155562294854
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,1024,0.005664888769388199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,512,0.004037333445416557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,1024,0.009962666365835402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,1536,0.010430222584141625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,768,0.004761777818202972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,256,0.003104888937539525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,768,0.009735110733244155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,128,0.002595555658141772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,64,0.0022693334354294673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,512,0.00942488925324546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,2048,32,0.0022604444788561925
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,256,0.009094222552246517
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,128,0.009114666945404476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,64,0.00847111145655314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,2048,32,0.008031110796663497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,16384,0.0768773357073466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,12288,0.057617776923709445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,16384,0.04692177640067207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,10240,0.047559112310409546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,8192,0.04717333449257744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,65536,0.15983200073242188
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,65536,0.28205421235826283
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,12288,0.03756977783309089
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,10240,0.020682666036817763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,7168,0.041278223196665444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,8192,0.016795555750528973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,6144,0.03632888860172696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,5120,0.027611555324660406
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,7168,0.015944888194402058
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,4096,0.016358221570650738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,6144,0.014954666296641031
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,3584,0.01461777753300137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,3072,0.012914666699038612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,4096,0.012786666552225748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,2560,0.011064888702498542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,3584,0.01240355521440506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,2048,0.009171555439631144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,5120,0.013931555880440606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,3072,0.011934222446547614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,1536,0.0074480002125104266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,1024,0.005561777700980504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,2560,0.011381333072980246
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,512,0.0036640001667870414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,1536,0.010339555641015371
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,768,0.004664888812435998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,1024,0.009826666778988307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,256,0.002963555562827322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,128,0.0024115554988384247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,64,0.002187555655837059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,512,0.009304888546466827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,2048,0.010833777487277985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,768,0.009650666680600908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,256,0.00904888908068339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1536,32,0.002179555594921112
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,128,0.009208888643317753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,64,0.008552889029184977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,16384,0.05183644427193535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1536,32,0.007824000385072496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,12288,0.03811911079618666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,10240,0.03247822324434916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,8192,0.012552888856993781
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,65536,0.19076889091067842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,16384,0.030082666211658057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,65536,0.11461333433787028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,12288,0.020917332834667627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,7168,0.008070222205585903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,8192,0.016748444901572335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,10240,0.01885511146651374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,6144,0.007344000041484833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,5120,0.006844444407357111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,4096,0.005964444329341252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,3584,0.0060159998635451
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,3072,0.0053102220926019884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,6144,0.014851555228233337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,4096,0.012772444221708508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,7168,0.01590222285853492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,2560,0.00545155546731419
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,3584,0.0123831108212471
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,5120,0.013888888888888888
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,1536,0.007280000381999546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,3072,0.011876444849703046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,2048,0.009100444614887238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,1024,0.00545866663257281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,2560,0.011374221907721626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,768,0.004517333375083076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,2048,0.010976889067225985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,512,0.00360000009338061
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,1536,0.010384000009960597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,256,0.0028400001012616684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,1024,0.009775111244784461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,128,0.0021600000974204806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,768,0.009577777650621202
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,512,0.009290667043791877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,64,0.0021226666867733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,1024,32,0.002128888956374592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,256,0.009000000026490953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,128,0.009222221871217092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,64,0.008438222110271454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,1024,32,0.007803555164072249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,16384,0.03965422179963853
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,12288,0.029616889026429918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,10240,0.009475555684831407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,16384,0.02477333280775282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,65536,0.1436515516704983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,7168,0.008007110820876228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,12288,0.020813332663642038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,65536,0.09932977623409694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,8192,0.008252444366614023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,10240,0.018851555056042142
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,6144,0.007259555160999298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,8192,0.016715556383132935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,4096,0.00612088872326745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,7168,0.015828443898095023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,6144,0.014808000789748298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,5120,0.0069431112044387394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,3072,0.005039111193683413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,5120,0.013891556196742587
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,3584,0.005423111220200856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,2560,0.004986666556861666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,4096,0.012724444270133972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,3584,0.012310222619109683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,2048,0.004473777694834603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,3072,0.01183644433816274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,1536,0.004414222306675381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,1024,0.005410666680998272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,768,0.004482666651407878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,2048,0.01073155552148819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,512,0.0037760001917680106
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,1536,0.010276444256305695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,1024,0.009728000395827824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,256,0.002744888886809349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,768,0.009460444251696268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,128,0.0021582223061058256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,512,0.009289777941173976
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,64,0.002130666747689247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,768,32,0.002132444539003902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,256,0.008880889250172509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,128,0.009183111290136972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,64,0.008526222573386298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,2560,0.011298666397730509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,768,32,0.007848000360859765
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,65536,0.09642399681939019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,16384,0.01407999959256914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,10240,0.007835555407736037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,12288,0.00889066689544254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,16384,0.0247662216424942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,8192,0.007155555817815993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,65536,0.08966577715343899
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,7168,0.006719999843173557
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,6144,0.006284444282452266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,10240,0.018872888551818002
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,12288,0.020807110601001315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,5120,0.0058168888919883305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,8192,0.01668711172209846
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,4096,0.005119110975000593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,7168,0.015847999188635085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,6144,0.014853333433469137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,3072,0.0047306666771570844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,3584,0.005119110975000593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,5120,0.013863110707865821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,2560,0.004655111167165968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,4096,0.012678222523795234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,2048,0.004254222330119875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,3584,0.012246222131782107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,1536,0.004201777693298128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,1024,0.005364444520738389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,3072,0.011858666936556498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,768,0.0047626665068997275
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,2560,0.011300444602966309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,2048,0.010743111371994019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,256,0.0027057776848475137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,512,0.0037253333462609183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,1024,0.009689778089523315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,1536,0.010249777800507015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,128,0.002132444539003902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,768,0.009526222116417354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,64,0.002060444404681524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,512,32,0.0020657777786254883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,512,0.009337777892748514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,256,0.008991999758614434
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,128,0.00923555592695872
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,16384,0.009955555200576782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,64,0.008361777497662438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,12288,0.008235555556085374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,65536,0.05363555418120491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,512,32,0.007879999776681265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,10240,0.0058622223635514574
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,8192,0.006793777975771163
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,16384,0.024684444069862366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,7168,0.005180444568395615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,65536,0.07654933134714763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,12288,0.02086488902568817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,6144,0.005897777775923411
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,5120,0.00543200017677413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,10240,0.018876444962289598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,8192,0.016686222619480558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,6144,0.014769777655601501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,3584,0.004855999930037392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,5120,0.013832000394662222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,7168,0.015771556231710646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,4096,0.004254222330119875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,3072,0.004420444369316101
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,4096,0.012707555459605323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,2560,0.004401777767472797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,3584,0.012210666305489011
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,2048,0.003955555458863576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,1536,0.007956444389290279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,1024,0.005842666659090254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,3072,0.011800000237094032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,2560,0.01129422254032559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,768,0.0047608887155850725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,2048,0.010694222317801582
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,512,0.0037084445357322693
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,1536,0.010244444840484196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,256,0.0026577777332729767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,1024,0.009708444277445475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,128,0.002114666625857353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,64,0.0020702222569121257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,768,0.009518221848540837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,512,0.009135999613338048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,256,32,0.0020862221717834473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,256,0.008820444345474243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,128,0.00925511121749878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,16384,0.006728000111050076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,65536,0.013707555830478668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,12288,0.005819555371999741
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,32,0.007898666792445714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,256,64,0.00849599970711602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,10240,0.005383111122581694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,8192,0.00489155575633049
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,16384,0.024643555283546448
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,12288,0.020816889074113634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,7168,0.004892444445027245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,65536,0.07207733392715454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,10240,0.018834667073355783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,8192,0.016713778177897137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,6144,0.004447999927732679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,5120,0.004416888786686791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,7168,0.015817777978049386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,4096,0.003998222036494149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,3584,0.016385777129067313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,5120,0.013835555149449242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,6144,0.014721777704026965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,3072,0.014306666122542487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,4096,0.012711111042234631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,3584,0.012246222131782107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,2560,0.012206222448084088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,2048,0.010084444450007545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,1536,0.00794666674402025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,3072,0.011741333537631564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,1024,0.0058142224119769196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,2560,0.011242667006121742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,768,0.004754666652944353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,2048,0.010761777559916178
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,512,0.003705777641799715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,256,0.0026337777574857077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,1536,0.010269333091047075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,128,0.0020888888587554297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,1024,0.009709333380063375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,768,0.00941066692272822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,512,0.009166222479608323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,32,0.002061333300338851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,256,0.008931555681758458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,128,64,0.002061333300338851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,128,0.009119999905427298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,65536,0.009336888790130615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,16384,0.006619555668698416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,64,0.008466666771305932
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,128,32,0.007799111306667328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,12288,0.005699555493063397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,10240,0.0053226666318045715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,8192,0.004744889007674323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,7168,0.03113599949412876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,65536,0.07209244701597425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,16384,0.02472355630662706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,12288,0.020812445216708712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,10240,0.018823999497625563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,8192,0.016692444682121277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,5120,0.022700443863868713
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,6144,0.02699822187423706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,4096,0.018477333916558158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,7168,0.01569422251648373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,3584,0.016395555602179635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,3072,0.014283556077215405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,6144,0.014774221513006421
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,5120,0.013760888742076026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,4096,0.012702222499582501
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,3584,0.012262221839692859
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,2560,0.012173333101802401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,2048,0.010060444474220276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,1536,0.00792266676823298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,3072,0.011676444775528379
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,1024,0.0058124446206622655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,768,0.004755555755562252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,2560,0.011241777903503843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,512,0.0036755556033717263
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,2048,0.010620444185203977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,256,0.0026480000880029467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,1536,0.010206222534179688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,1024,0.009662222531106737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,128,0.0020942222326993942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,64,0.0020791112134853997
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,512,0.009159999589125315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,64,32,0.0020622221959961788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,256,0.008928889201747047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,768,0.009385777844323052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,65536,0.009083555804358589
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,128,0.009148444566461775
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,64,0.008427555362383524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,16384,0.006492444624503453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,12288,0.005588444570700328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,64,32,0.007928888830873702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,10240,0.005127999931573868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,8192,0.03538577755292257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,16384,0.024338665935728285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,65536,0.07094844182332356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,7168,0.031073778867721558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,12288,0.02049511174360911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,10240,0.018637332651350234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,6144,0.026672000686327618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,5120,0.022670222653283015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,8192,0.016573333077960543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,4096,0.018438221679793466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,7168,0.015607110328144498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,3584,0.016359110673268635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,6144,0.014544000228246054
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,3072,0.01423111061255137
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,5120,0.013678222066826291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,2560,0.012137778103351593
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,4096,0.012554667062229581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,2048,0.010046222143703038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,3584,0.012179555164443122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,1536,0.007898666792445714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,3072,0.011678222152921887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,1024,0.005843555761708154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,2560,0.011149333582984077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,768,0.0047377778424157035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,2048,0.01069777790043089
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,512,0.003628444340493944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,1536,0.010088000032636855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,256,0.002625777696569761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,1024,0.009691555466916826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,128,0.0020755556308560902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,768,0.009474666582213508
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,64,0.002050666759411494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,512,0.009209777745935652
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,4,32,32,0.002040000011523565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,256,0.008758222063382467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,128,0.009092444347010719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,64,0.008393777741326226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,4,32,32,0.007888000044557784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,10240,1.161687109205458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,12288,1.402343961927626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,8192,1.5132640202840169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,16384,1.8453511132134333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,10240,1.8448772430419922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,8192,0.8959937625461154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,12288,2.3805768754747176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,7168,0.7517822053697375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,6144,0.678028424580892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,7168,1.2927697499593098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,16384,2.951102150811089
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,5120,0.9247342215643989
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,5120,0.5536115434434679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,6144,1.1239164140489366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,4096,0.7408844629923502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,4096,0.43881244129604763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,3072,0.30827199088202584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,3072,0.5563102298312718
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,3584,0.3888400130801731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,2048,0.37273600366380477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,2560,0.4645893308851454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,2560,0.28789154688517254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,3584,0.7087955474853516
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,2048,0.23157689306471083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,768,0.1417671177122328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,1536,0.16652977466583252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,512,0.0944675538274977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,1024,0.20603556103176543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,256,0.04675733380847507
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,768,0.10319555468029445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,1536,0.3059937689039442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,512,0.08316000302632649
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,128,0.08204178015391032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,32,0.004043555508057277
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,128,0.011367110742463006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,65536,64,0.0064328888224230866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,32,0.08060444725884332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,64,0.0790133343802558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,1024,0.12779377566443548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,65536,256,0.07876977655622694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,16384,0.4528293079800076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,12288,0.365339544084337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,10240,0.3032631079355876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,10240,0.5099564658270942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,12288,0.6621804237365723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,16384,0.8560133510165744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,8192,0.38301955329047305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,6144,0.2868115637037489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,7168,0.21913511223263213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,8192,0.24788088268703887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,7168,0.35875643624199766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,6144,0.1758240063985189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,5120,0.25699377059936523
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,4096,0.13208888636695013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,3584,0.1807840002907647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,5120,0.15885244475470647
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,3072,0.14156177308824328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,4096,0.2061413394080268
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,65536,1.6161502202351887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,2048,0.09484710958268906
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,2560,0.1301600005891588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,1536,0.07139999998940362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,2560,0.08601599931716919
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,3072,0.0976053343878852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,2048,0.06687644455167982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,3584,0.11836177772945827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,768,0.036905778778923884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,1536,0.04789333211051094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,1024,0.05340622199906243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,256,0.0041315555572509766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,512,0.008027555214034187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,512,0.011955555942323474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,256,0.011642666326628791
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,64,0.0029253332565228143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,128,0.010249777800507015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,768,0.01346933344999949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,128,0.0031742221779293488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,32,0.0028373334142896864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,1024,0.0339937773015764
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,64,0.009628444082207149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,16384,32,0.009505777723259395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,16384,0.3273022174835205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,12288,0.5035146607293023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,12288,0.2782506677839491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,10240,0.35319288571675617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,16384,0.6432035764058431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,16384,65536,2.9471884833441844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,8192,0.1711768839094374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,7168,0.1489351060655382
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,65536,1.215327156914605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,8192,0.32845600446065265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,10240,0.22832889027065703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,7168,0.28496623039245605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,6144,0.2299262152777778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,6144,0.1288693348566691
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,5120,0.1859733396106296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,4096,0.1558257738749186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,4096,0.09974666436513265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,5120,0.12006221877204047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,3584,0.14534578058454725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,3584,0.08697333600785996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,3072,0.07387555307812162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,3072,0.1240275568432278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,2560,0.1038551131884257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,2560,0.062136001057095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,1024,0.04273777869012621
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,2048,0.05130044288105435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,2048,0.0831128888660007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,1536,0.06276444594065349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,512,0.005487111293607288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,1536,0.04045244389110141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,768,0.032807999187045626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,256,0.004011555678314633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,65536,2.211043463812934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,128,0.002918222298224767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,1024,0.01254488858911726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,32,0.002653333254986339
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,768,0.012327111429638334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,256,0.010891555911964841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,512,0.011224889092975192
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,32,0.00888888869020674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,12288,64,0.00272088891102208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,64,0.009072889056470659
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,12288,128,0.009920888476901585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,16384,0.2754906548394097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,12288,0.21189599566989473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,16384,0.4642648696899414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,10240,0.29125955369737416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,12288,0.4220746623145209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,8192,0.2402391168806288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,10240,0.17452800273895264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,8192,0.14041154914432102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,7168,0.20486666096581352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,7168,0.13424711757236057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,6144,0.1815191109975179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,6144,0.10882311397128636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,4096,0.11787822511461045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,5120,0.09361510806613499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,3584,0.103401780128479
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,5120,0.16179111268785265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,3584,0.06618577904171415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,65536,1.0131830639309354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,4096,0.0813902219136556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,3072,0.09844799836476643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,2048,0.06003644731309679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,2560,0.048453334305021495
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,1536,0.050514668226242065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,3072,0.05752444267272949
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,1024,0.031155556440353394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,2048,0.040050665537516274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,1536,0.019834667444229126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,768,0.006736889067623351
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,512,0.005346666607591841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,256,0.0034942221310403612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,2560,0.08325066831376818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,128,0.002761777697337998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,1024,0.011971555650234222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,768,0.011870221959220039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,64,0.002628444383541743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,32,0.002591111179855135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,256,0.010816000401973724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,64,0.008955555657545725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,32,0.00868622213602066
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,128,0.009581333233250512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,10240,512,0.01108888867828581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,10240,65536,1.8432977464463975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,16384,0.2152480019463433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,12288,0.16530311107635498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,12288,0.31122488445705837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,10240,0.13812888993157282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,10240,0.26270222663879395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,16384,0.42021600405375165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,8192,0.19211199548509386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,8192,0.11554488870832656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,7168,0.16449688540564641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,5120,0.07475733094745211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,6144,0.1454346709781223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,4096,0.0950124462445577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,6144,0.09604533513387044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,65536,0.7804329130384656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,5120,0.13264089160495335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,3584,0.08327288760079278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,4096,0.06005511018964979
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,3072,0.07173066669040255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,3584,0.0528400010532803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,3072,0.046032889021767505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,2560,0.03947910997602675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,1536,0.03696533375316196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,7168,0.1107600000169542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,1024,0.014663999279340109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,2048,0.05467111203405592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,768,0.006065777606434292
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,1536,0.013038222160604266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,512,0.004770666774776247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,2560,0.06753689050674438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,1024,0.012112889024946423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,256,0.0035662220584021676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,768,0.011821333732869891
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,128,0.0027351110345787476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,512,0.011190222369299995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,64,0.0026791110220882627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,2048,0.031228442986806233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,256,0.011115555961926779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,65536,1.4758453369140625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,8192,32,0.0026764445420768526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,128,0.009852444132169088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,64,0.009487999810112847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,8192,32,0.00924533357222875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,16384,0.1861884461508857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,12288,0.27096978823343915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,16384,0.3430222140418158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,10240,0.204858660697937
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,12288,0.153674668735928
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,10240,0.11930933263566758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,8192,0.17046756214565703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,7168,0.14469333489735922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,6144,0.12790489196777344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,65536,0.6756338013543023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,7168,0.0952577789624532
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,5120,0.10349600182639228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,6144,0.07581066422992282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,5120,0.06353333261277941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,8192,0.10596266720030044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,4096,0.09437066978878444
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,3072,0.06336622105704413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,4096,0.055330667230818004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,3584,0.08235733376608954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,2560,0.059354669517940946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,3072,0.043528887960645885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,3584,0.049716442823410034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,2048,0.048134220971001514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,2048,0.015863999724388123
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,1024,0.007161777880456712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,2560,0.034287999073664345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,768,0.006112888869312074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,1536,0.036575112077924944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,65536,1.2910017437405057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,512,0.004761777818202972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,1536,0.012416889270146688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,256,0.003589333345492681
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,128,0.0026640000028742682
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,1024,0.011720889144473605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,768,0.01143555591503779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,512,0.010783111055692038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,64,0.002630222174856398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,7168,32,0.0026515554636716843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,64,0.0092257774538464
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,32,0.00902844468752543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,128,0.00980711148844825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,7168,256,0.010649777948856354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,12288,0.23529511027865938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,16384,0.1607582171758016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,12288,0.13115999433729383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,16384,0.3079386552174886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,10240,0.1981359985139635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,8192,0.14775110615624323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,65536,0.581432024637858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,10240,0.1034311122364468
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,5120,0.0891724427541097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,7168,0.1396133369869656
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,7168,0.08075377676222059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,6144,0.12426311439938015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,4096,0.07257599963082208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,6144,0.06593422094980876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,65536,1.106833775838216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,8192,0.0918160014682346
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,3584,0.07086133294635348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,3584,0.04392177859942118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,2560,0.04586311181386312
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,3072,0.0612915555636088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,5120,0.05606310897403293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,3072,0.03561866614553664
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,1536,0.028346667687098186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,4096,0.04871822065777249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,1024,0.006834666762087081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,2560,0.02239022155602773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,512,0.004512888689835866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,2048,0.04178222351604038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,1024,0.011164444188276926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,1536,0.01187999960449007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,256,0.0035048888789282907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,2048,0.0125608891248703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,768,0.011195555329322815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,128,0.0026106666773557663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,64,0.002570666579736604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,512,0.010296888649463654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,256,0.010296000374688042
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,32,0.0025724443710512584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,128,0.009461333354314169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,64,0.008996444443861643
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,6144,32,0.008831111093362173
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,6144,768,0.005669333454635408
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,16384,0.13305777973598903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,16384,0.23786489168802896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,10240,0.1627688937717014
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,12288,0.10353066523869832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,12288,0.20991199546390107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,10240,0.08720266819000244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,8192,0.12387200196584065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,7168,0.10396711031595866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,65536,0.48778311411539715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,6144,0.09251377979914348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,8192,0.07648888561460707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,7168,0.06306133005354139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,5120,0.07494755585988362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,4096,0.06121777825885349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,3584,0.05312888820966085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,3072,0.04627644353442722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,6144,0.0593128866619534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,5120,0.05061777763896518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,4096,0.042267554336123996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,65536,0.9235511355929904
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,2048,0.03531644410557217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,2560,0.03875199953715006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,1536,0.00945955514907837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,2560,0.012631110846996307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,1024,0.006457777900828256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,2048,0.01200177768866221
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,3584,0.03318399853176541
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,768,0.005409777578380372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,3072,0.01957155598534478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,1536,0.011401777466138205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,1024,0.010786666638321347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,256,0.003232888877391815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,512,0.004302222281694412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,768,0.010970667004585266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,64,0.002529777793420686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,512,0.01015822258260515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,256,0.010255999863147736
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,128,0.0025351111673646504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,5120,32,0.002542222125662698
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,64,0.008651555412345463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,32,0.00867022242810991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,5120,128,0.00928177767329746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,16384,0.10775199863645767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,16384,0.18990488847096762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,12288,0.15377777152591282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,12288,0.08442488643858169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,8192,0.0995217760403951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,7168,0.08399199777179295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,8192,0.058111998769972056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,10240,0.13332266277737087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,65536,0.39113333490159774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,6144,0.07428799735175239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,4096,0.04950399862395393
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,5120,0.06769955820507474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,6144,0.0444524453745948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,65536,0.740388446384006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,10240,0.07591644260618421
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,5120,0.042282667424943715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,7168,0.05574311150444878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,3584,0.013471999930010902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,2560,0.03498844305674235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,2048,0.012665777570671506
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,3072,0.012447111308574677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,4096,0.023018666439586218
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,1536,0.008087111016114553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,2560,0.011894221935007306
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,1024,0.006029333505365584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,3072,0.04221333397759331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,768,0.005028444445795483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,2048,0.011448000040319232
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,512,0.0041368889311949415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,3584,0.04806222187148201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,1536,0.011046222514576383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,1024,0.010458666417333813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,256,0.003043555551105075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,64,0.002456888970401552
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,768,0.010370666782061258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,128,0.002487999904486868
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,4096,32,0.0024666666156715816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,512,0.009881777895821465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,128,0.008834666675991481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,256,0.009752000371615091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,64,0.008280889027648503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,4096,32,0.00830488900343577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,10240,0.10451910893122356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,12288,0.1392399999830458
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,16384,0.17226044336954752
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,12288,0.07396355602476332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,10240,0.06217066446940104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,16384,0.10405421919292873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,8192,0.08748177687327068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,7168,0.0742151141166687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,65536,0.34334490034315324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,7168,0.044998221927218966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,6144,0.06577066580454509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,8192,0.050473776128556996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,6144,0.03984533415900336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,4096,0.04385866721471151
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,5120,0.057710223727756076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,3584,0.03813955518934462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,3072,0.0336168905099233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,4096,0.014039110806253223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,65536,0.648823102315267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,2560,0.02419822249147627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,5120,0.036842667394214206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,2048,0.010033778018421596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,3072,0.012248888611793518
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,3584,0.012711111042234631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,2560,0.01183999992079205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,1536,0.007994666695594788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,1024,0.005939555664857228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,768,0.004962666581074397
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,2048,0.01130488928821352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,1536,0.010837333069907295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,256,0.0032417778339650896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,1024,0.010281778044170803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,768,0.010318222145239512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,512,0.004106666478845808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,128,0.00244711111817095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,32,0.0023973333752817577
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,256,0.009555555880069733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3584,64,0.002432000098956956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,128,0.008700444466537898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,512,0.00978488889005449
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,64,0.008207110895050896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3584,32,0.008231110870838165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,16384,0.1424542268117269
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,12288,0.12029422654045953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,10240,0.09047200282414754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,16384,0.08353244596057469
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,12288,0.06388711267047457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,8192,0.07613777452045016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,7168,0.0645573337872823
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,10240,0.05375644564628601
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,7168,0.03985777828428481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,8192,0.0440995560752021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,65536,0.30865600374009877
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,4096,0.03824266791343689
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,5120,0.015413333972295126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,65536,0.5564355320400661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,6144,0.03455555438995361
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,4096,0.013172444370057849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,6144,0.05743200249142117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,5120,0.04651555418968201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,3072,0.02954044606950548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,2560,0.014807111687130399
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,2048,0.009877333210574256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,3584,0.03337511089113023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,3584,0.012624888784355588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,1024,0.0058266665372583605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,3072,0.012132444315486483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,2560,0.011678222152921887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,1536,0.007868444754017724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,768,0.004907555464241239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,512,0.004063999901215236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,1536,0.010650667051474253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,256,0.0032355555643637977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,1024,0.01018488903840383
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,768,0.010210666391584609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,64,0.0023475556323925653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,128,0.002416888872782389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,2048,0.011051555474599203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,512,0.009533333281675974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,3072,32,0.0023511110080613033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,256,0.009536888864305284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,128,0.00867377801073922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,64,0.008064889245563084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,16384,0.1271102163526747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,12288,0.09592533111572266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,16384,0.06996177964740329
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,12288,0.054914666546715625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,10240,0.07874666982226901
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,3072,32,0.008036444584528605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,65536,0.2515244483947754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,8192,0.06532800197601318
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,7168,0.054937776592042714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,10240,0.0457857780986362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,7168,0.033210668298933245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,5120,0.040503111150529646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,8192,0.03824355536037021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,4096,0.03293422195646498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,65536,0.4657111167907715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,3584,0.027062222361564636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,5120,0.013989332649442883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,3072,0.013202666408485837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,2560,0.011377777490350934
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,4096,0.013047110703256396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,3584,0.012448000411192576
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,6144,0.05488711264398363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,6144,0.018290667070282828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,2048,0.00941422250535753
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,1536,0.00758844448460473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,3072,0.011872000164455838
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,1024,0.005716444303592046
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,768,0.005126222140259213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,512,0.0038355555799272326
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,2560,0.011556444068749746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,2048,0.011001778145631155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,256,0.002918222298224767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,128,0.0023804443577925363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,1024,0.009944000177913243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,1536,0.010557333628336588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,512,0.009483555952707926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,64,0.002321777865290642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,256,0.009436444275909001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2560,32,0.0023084444304307303
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,768,0.009962666365835402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,128,0.008602666358153025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,64,0.007831110722488826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2560,32,0.007987555530336168
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,12288,0.08361599842707317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,10240,0.06653866502973768
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,12288,0.04637777805328369
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,16384,0.10414577855004205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,65536,0.20592089494069418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,8192,0.05444799860318502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,16384,0.06387288702858819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,7168,0.04668710960282219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,65536,0.37344710032145184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,8192,0.020489777127901714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,10240,0.038374221987194486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,5120,0.03479111194610596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,6144,0.04112622141838074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,7168,0.01607911123169793
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,4096,0.022150221798155043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,3584,0.014682667122946845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,3072,0.012882666455374824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,5120,0.013903111219406128
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,6144,0.014876445134480795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,3584,0.012306667036480375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,4096,0.012827555338541666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,2048,0.00924622184700436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,2560,0.011471110913488599
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,3072,0.011864888999197217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,2048,0.01092088884777493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,1536,0.0074471111098925276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,768,0.0046951112647851305
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,512,0.0037493333220481873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,1024,0.005608888963858287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,2560,0.011062222222487131
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,1536,0.010395555860466426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,256,0.003119999956753519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,1024,0.009905777871608734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,128,0.002299555473857456
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,768,0.009927111367384592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,64,0.002247111043996281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,2048,32,0.002248888835310936
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,512,0.00925688859489229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,256,0.009320889082219865
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,64,0.00795022232664956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,32,0.007975111405054728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,16384,0.07690133651097615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,12288,0.05762310822804769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,16384,0.04634666774008009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,10240,0.04750222298834059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,2048,128,0.008551999926567078
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,65536,0.15843199359046087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,12288,0.03693422343995836
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,65536,0.281821330388387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,8192,0.038586666186650596
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,10240,0.019352000620630052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,7168,0.041831112570232816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,6144,0.035692443450291954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,5120,0.024866667058732774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,8192,0.016607999801635742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,6144,0.014727999766667684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,3584,0.01457333399189843
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,4096,0.016336000627941556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,5120,0.013807999591032663
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,3072,0.012792000340090858
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,4096,0.01276266657643848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,7168,0.01575644479857551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,2560,0.010983111129866706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,1536,0.007380444142553542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,3584,0.012288000020715924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,2560,0.011351111034552256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,1024,0.005558222118351195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,2048,0.010832889212502373
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,2048,0.009151111046473185
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,768,0.004609777695602841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,3072,0.011828444070286222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,512,0.003908444609906939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,256,0.002963555562827322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,1536,0.010399999717871347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,1024,0.009773333039548662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,128,0.0022435554613669715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,768,0.009859555297427708
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,64,0.0021991110924217436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1536,32,0.002191999927163124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,512,0.009143999881214565
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,256,0.009401777552233802
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,128,0.00850133349498113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,64,0.007867555651399825
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1536,32,0.00790666706032223
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,16384,0.04959911108016968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,12288,0.038167999850379095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,16384,0.02825688984658983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,12288,0.020635555187861126
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,65536,0.1904133293363783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,65536,0.11509333716498481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,10240,0.03246577911906772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,8192,0.01071911139620675
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,6144,0.007234666910436418
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,10240,0.01863911085658603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,8192,0.016584000653690763
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,5120,0.006712889091836081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,6144,0.01478666729397244
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,4096,0.005879999862776862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,7168,0.015678221980730694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,7168,0.007862221863534715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,3584,0.005966222120655908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,2560,0.0053377776510185665
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,5120,0.013780444032616086
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,4096,0.012703110774358114
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,2048,0.009091555244392818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,3072,0.005335111171007156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,1536,0.007275555696752336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,3072,0.011778666741318174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,3584,0.012342222034931183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,2560,0.011326221956147088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,1024,0.005420444326268301
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,2048,0.010777778095669217
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,512,0.003768889026509391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,256,0.0027253333893087176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,1536,0.010341333018408881
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,1024,0.009732444253232744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,512,0.009111111362775167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,128,0.0021608887861172357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,64,0.0021484444538752237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,256,0.009259555902745988
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,32,0.0021253333737452826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,1024,768,0.004843555390834808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,128,0.00849333322710461
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,768,0.009914666414260864
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,64,0.007837333612971837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,16384,0.040574222803115845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,1024,32,0.007816000117195977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,12288,0.03132889005872939
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,65536,0.1430035564634535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,10240,0.009900444083743626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,8192,0.00812000036239624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,16384,0.02439822256565094
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,65536,0.10021777947743733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,12288,0.02055999967787001
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,7168,0.007912000020345053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,10240,0.018608889646000333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,6144,0.007126222054163615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,8192,0.016535111599498324
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,5120,0.006909333169460297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,4096,0.006045333213276333
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,3584,0.0053839998112784485
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,6144,0.014732443624072604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,3072,0.004990222139490975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,7168,0.015673778123325773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,2560,0.004991111242108875
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,2048,0.004471999903519948
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,4096,0.012666666673289405
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,3584,0.012186666329701742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,3072,0.011738667057620155
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,1536,0.004385777645640903
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,5120,0.013697778185208639
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,1024,0.005902222047249476
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,2560,0.011266666981909009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,768,0.004821333206362194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,512,0.0037848887344201407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,1536,0.010251555177900527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,2048,0.01070666644308302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,256,0.002749333365095986
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,1024,0.009709333380063375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,768,0.009700444837411245
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,128,0.0021502222451898786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,64,0.0021315556433465746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,768,32,0.002104000084929996
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,512,0.009128000173303816
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,256,0.00925066653225157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,128,0.008423999779754216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,16384,0.015505777464972602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,64,0.00795999997191959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,768,32,0.007898666792445714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,12288,0.008784888519181145
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,10240,0.007800000409285228
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,65536,0.0962382223871019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,8192,0.007131555842028723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,16384,0.02437066700723436
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,7168,0.006690666493442323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,65536,0.08829244640138413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,12288,0.020580444071027968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,6144,0.006210666563775804
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,10240,0.01866311166021559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,5120,0.005327111317051782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,8192,0.01649599936273363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,4096,0.005111999809741974
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,7168,0.015619556109110514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,3584,0.005065777649482091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,3072,0.004659555438492033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,5120,0.013747555514176687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,4096,0.01258933295806249
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,2560,0.0046302220887608
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,3584,0.012171555724408893
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,6144,0.014679999815093147
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,1536,0.007977777885066139
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,3072,0.011750222080283694
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,2048,0.004222222086456087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,1024,0.0058515556156635284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,2560,0.011244444383515252
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,768,0.0047591109242704184
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,2048,0.010690666735172272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,512,0.003718222181002299
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,1536,0.010281778044170803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,256,0.0026355555488003623
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,1024,0.009675555759006077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,128,0.002100444502300686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,768,0.009655999640623728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,64,0.002061333300338851
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,512,0.009109333157539368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,512,32,0.0020915555457274118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,256,0.009378666679064432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,16384,0.009933333430025313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,65536,0.053454223606321544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,64,0.007827555967701806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,32,0.00794488853878445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,512,128,0.008463111188676622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,12288,0.006041777630647023
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,10240,0.005598222215970357
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,8192,0.006775110960006714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,7168,0.005102222164471944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,16384,0.024338665935728285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,12288,0.020544000797801547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,65536,0.07535911268658109
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,10240,0.01853155593077342
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,6144,0.004658666749795278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,5120,0.004625777817434735
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,8192,0.01657511128319634
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,4096,0.0041964443193541635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,7168,0.015638222297032673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,3584,0.01641244524055057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,3072,0.014320888453059726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,2560,0.0122079998254776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,6144,0.014698666003015308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,5120,0.013691555294725629
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,4096,0.012620444099108378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,3584,0.012279999752839407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,2048,0.010079999764760336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,1536,0.007967111137178209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,3072,0.011742221812407175
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,2560,0.011180444724029966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,768,0.004787555585304896
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,1024,0.0058417779703934984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,2048,0.010667555862002902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,512,0.003689777933888965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,256,0.0026524443593290118
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,1536,0.010235555469989777
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,1024,0.009717333647939894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,128,0.0020888888587554297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,768,0.00978577799267239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,64,0.0020737778395414352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,256,32,0.002058666613366869
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,256,0.009328888522254096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,65536,0.014919999572965833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,128,0.00848444468445248
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,64,0.007927999728255803
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,16384,0.006717333363162146
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,32,0.007826666865083907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,12288,0.0058062221441004015
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,10240,0.0053297777970631914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,256,512,0.009039999710188972
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,8192,0.004870222260554631
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,65536,0.07117422421773274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,16384,0.024327110913064744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,12288,0.02055199940999349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,7168,0.03118577930662367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,10240,0.018592889110247295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,6144,0.02700177828470866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,5120,0.022749332918061152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,8192,0.016532444291644625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,7168,0.015666666958067153
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,4096,0.0185244447655148
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,3584,0.016401777664820354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,6144,0.014676445060306124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,3072,0.014307555225160388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,4096,0.01256533298227522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,2560,0.012192000117566852
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,5120,0.013712888790501488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,3584,0.01218844453493754
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,2048,0.010081777969996134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,1536,0.00794933322403166
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,3072,0.011724444727102915
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,2560,0.011218667030334473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,1024,0.0058133333093590206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,768,0.004751111070315043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,512,0.0036880001425743103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,2048,0.010626667075686984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,1536,0.0101742222905159
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,1024,0.009731555150614845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,256,0.002663111107216941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,128,0.002100444502300686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,768,0.009743111001120673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,64,0.0020755556308560902
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,512,0.00909511082702213
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,128,32,0.0020657777786254883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,256,0.009173333644866943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,65536,0.00923466682434082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,16384,0.006515555497672822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,128,0.008480889101823172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,64,0.007888000044557784
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,128,32,0.007868444754017724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,12288,0.005621333503060871
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,10240,0.0052284445199701525
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,65536,0.07119199964735243
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,8192,0.035517334938049316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,16384,0.024327110913064744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,12288,0.02052977846728431
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,7168,0.03118844495879279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,10240,0.018556444181336295
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,8192,0.01645511057641771
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,6144,0.02699111070897844
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,5120,0.02274399995803833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,4096,0.01847111185391744
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,7168,0.015506666567590503
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,6144,0.014648889501889547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,3584,0.01641155613793267
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,3072,0.014321777555677625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,5120,0.013639999760521783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,2560,0.012198222180207571
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,4096,0.01256444470749961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,2048,0.010081777969996134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,3584,0.012123555772834353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,1536,0.007921777665615082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,3072,0.011576000187132092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,1024,0.0058240000572469496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,2560,0.011187555889288584
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,768,0.004746666798988978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,512,0.0037004442678557504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,2048,0.010662222074137794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,1536,0.01015644437736935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,256,0.0026417778184016547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,128,0.0021066665649414062
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,1024,0.009553777674833933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,64,0.0020737778395414352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,768,0.009539555344316695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,512,0.009142222503821055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,64,32,0.0020666666742828158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,256,0.009264888862768808
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,128,0.008408889174461365
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,65536,0.008976000050703684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,16384,0.006456888798210356
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,12288,0.005560889012283749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,64,0.007824000385072496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,64,32,0.007851555943489075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,10240,0.005101333475775189
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,8192,0.03512888815667894
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,65536,0.06996888584560819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,16384,0.02405511173937056
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,7168,0.03077510992685954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,12288,0.02015822297996945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,10240,0.018333334061834548
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,6144,0.026911111341582403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,8192,0.016362667083740234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,5120,0.022481777601771887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,4096,0.018434666925006442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,3584,0.016366221838527255
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,7168,0.01540177729394701
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,6144,0.014540443817774454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,5120,0.01352266636159685
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,3072,0.01425244410832723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,2560,0.012033777932325998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,4096,0.012518222133318583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,3584,0.012011555333932241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,2048,0.009935999910036722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,1536,0.007875555919276344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,3072,0.011584888729784222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,1024,0.005745777653323279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,2560,0.011134222149848938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,768,0.0047191112405723995
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,512,0.0036319999231232535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,2048,0.010609777437316047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,1536,0.010127999716334874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,256,0.0026071110947264563
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,1024,0.009548444714811113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,128,0.002069333361254798
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,64,0.0020595555090241963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,512,0.009159111314349705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,2,32,32,0.0024684444069862366
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,768,0.009678222239017487
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,256,0.009030222064918941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,128,0.008456000023418004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,64,0.00776800016562144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,2,32,32,0.007743111087216272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,10240,1.1606399748060439
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,12288,1.4025359683566625
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,16384,1.845171610514323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,10240,1.793032964070638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,8192,1.4323867162068684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,12288,2.1515208350287542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,8192,0.8938062455919055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,7168,0.7594390975104438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,6144,0.6184150907728407
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,5120,0.5073386828104655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,16384,2.8676098717583547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,6144,1.077111138237847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,7168,1.4123848809136286
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,5120,0.8972116046481662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,4096,0.40398844083150226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,3584,0.3892310990227594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,3072,0.5592702229817709
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,2560,0.463864008585612
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,3072,0.33679911825391984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,2048,0.3699493408203125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,2560,0.2832888762156169
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,1536,0.27729956309000653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,3584,0.7336257828606499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,4096,0.8397928873697916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,2048,0.23255554835001627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,1536,0.16643910937839085
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,512,0.11052355501386855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,1024,0.22328445646497938
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,768,0.16872444417741564
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,512,0.08431466420491536
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,256,0.04904266529613071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,128,0.025390222668647766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,64,0.006457777900828256
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,65536,32,0.004011555678314633
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,1024,0.12829421626196966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,64,0.07921333445443048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,32,0.07987378040949504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,128,0.08150666952133179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,768,0.10130488872528076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,65536,256,0.0789688891834683
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,16384,0.44449154535929364
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,12288,0.36745421091715497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,12288,0.5418151219685873
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,10240,0.303239107131958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,10240,0.5082604620191786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,16384,0.719537787967258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,8192,0.36018755700853133
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,7168,0.31600978639390737
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,8192,0.24738666746351454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,6144,0.17592267195383707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,7168,0.2191386620203654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,6144,0.3062346776326497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,5120,0.22649243142869738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,5120,0.14786756038665771
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,3584,0.17902221944597033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,4096,0.20424177911546496
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,3584,0.11907466252644856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,4096,0.13283644782172307
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,3072,0.10428266392813788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,3072,0.15403378009796143
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,2560,0.12811733616722956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,65536,1.6149430804782445
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,2560,0.08618933624691433
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,2048,0.06658489174313016
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,1024,0.0484862228234609
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,1536,0.08039733436372545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,1024,0.030631999174753826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,1536,0.052192889981799655
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,2048,0.10680711269378662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,768,0.04079822368092007
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,128,0.0031724443866146933
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,768,0.01346577786737018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,512,0.012587555580668978
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,64,0.004763555609517627
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,512,0.01536977787812551
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,256,0.011038222246699862
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,256,0.015286222100257874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,64,0.009456000394291347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,32,0.009554666777451834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,32,0.0024462222225136226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,16384,128,0.009865778187910715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,16384,0.3265644444359673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,12288,0.25169422891404897
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,10240,0.33870400322808164
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,12288,0.45936001671685117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,16384,0.6098640229966905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,10240,0.2085182269414266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,8192,0.271034664577908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,8192,0.17100266615549722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,16384,65536,2.9703990088568797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,6144,0.2044373353322347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,7168,0.2679351170857747
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,7168,0.16199910640716553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,5120,0.17055466439988878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,5120,0.11141244570414226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,65536,1.2158586714002821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,4096,0.13654843966166177
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,6144,0.14120266172620985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,4096,0.0935795572068956
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,3072,0.10315377844704522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,3584,0.11973510848151313
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,3072,0.06854755348629422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,2560,0.057486222849951855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,1536,0.05439022183418274
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,2560,0.09669511185752021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,2048,0.08097866508695814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,3584,0.08772266573376125
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,768,0.03490666548411051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,1024,0.012500444220172035
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,1536,0.03725511166784499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,512,0.00673333348499404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,2048,0.05084266596370273
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,1024,0.04155822263823615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,768,0.012329777909649743
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,512,0.011584888729784222
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,128,0.0063573333124319715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,256,0.00850133349498113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,256,0.010289777484205034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,64,0.003620444486538569
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,32,0.002373333399494489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,64,0.0088746663596895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,128,0.009389333426952362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,12288,32,0.008990222381220924
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,12288,65536,2.167531543307834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,12288,0.21092000272538927
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,10240,0.2827226585812039
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,16384,0.2741759883032905
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,12288,0.36582310994466144
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,16384,0.4511422051323785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,10240,0.17431288295321992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,8192,0.2264311048719618
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,8192,0.14191110928853354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,7168,0.12440710597568089
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,7168,0.2099484470155504
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,6144,0.17086577415466309
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,5120,0.14268444644080266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,65536,1.012324439154731
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,4096,0.11426044834984674
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,5120,0.09401155842675103
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,3584,0.10013688935173883
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,3072,0.08643733130560981
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,3584,0.0656800005171034
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,6144,0.11857422192891438
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,4096,0.08161511023839314
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,2560,0.07531378004286024
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,3072,0.05685777796639336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,2560,0.048228445980283946
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,1024,0.03127022253142463
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,1536,0.04581155710750156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,768,0.006776000062624614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,2048,0.06802311208513048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,2048,0.04398222102059258
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,512,0.0052586665583981415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,1024,0.01201777739657296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,768,0.011991110940774282
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,256,0.010651555326249866
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,128,0.0036915557252036203
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,512,0.011384888655609555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,65536,1.807544920179579
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,64,0.0035235554807715942
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,1536,0.01866755551762051
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,64,0.008777778181764815
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,10240,32,0.0023253332409593794
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,128,0.009129777550697327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,256,0.00998488896422916
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,10240,32,0.008768888811270395
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,10240,0.22682576709323457
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,12288,0.1657831139034695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,10240,0.1392515500386556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,16384,0.21610666645897758
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,16384,0.36137599415249294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,12288,0.2911493248409695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,8192,0.11676266458299424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,7168,0.10490222109688653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,6144,0.137170672416687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,7168,0.1597422228919135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,8192,0.20464711719089082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,6144,0.09760000308354695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,4096,0.09184533357620239
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,5120,0.11463911003536648
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,4096,0.05922489033805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,5120,0.07478666967815824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,3584,0.09068177806006537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,3584,0.05304000112745497
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,2560,0.06096888913048638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,2048,0.055042667521370776
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,65536,0.8652453422546387
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,2560,0.03975644376542833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,3072,0.04557244314087761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,1536,0.03719822234577603
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,768,0.006058666855096817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,512,0.004791999856630961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,2048,0.029472000069088403
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,3072,0.08221244149737887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,1024,0.01186222251918581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,1536,0.01311644415060679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,1024,0.0284862220287323
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,256,0.0033395555284288195
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,512,0.011628444823953839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,128,0.0032879999942249725
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,256,0.010236444572607676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,64,0.009389333426952362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,128,0.009535999761687385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,32,0.0022053333620230355
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,768,0.011935110721323224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,64,0.002990222225586573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,8192,32,0.009279111193286048
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,8192,65536,1.6292862362331813
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,16384,0.186170670721266
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,16384,0.3166320059034559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,12288,0.1424080000983344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,10240,0.1988977723651462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,12288,0.27001065678066677
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,8192,0.1592879957622952
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,10240,0.12042578061421712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,8192,0.1009404460589091
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,7168,0.14009866449568006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,6144,0.12053067154354519
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,65536,0.6735502349005805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,7168,0.09558666414684719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,6144,0.07579644521077473
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,5120,0.1007137762175666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,4096,0.08079466554853651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,4096,0.051240000459882945
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,3584,0.04550844430923462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,3584,0.07972711324691772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,3072,0.04019377628962199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,2560,0.05386844608518812
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,3072,0.06512977679570515
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,5120,0.067358222272661
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,2560,0.034522665871514216
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,1536,0.01257333325015174
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,1536,0.036792887581719294
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,1024,0.025262221693992615
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,768,0.005853333406978183
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,2048,0.04853866828812493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,65536,1.2740604612562392
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,512,0.004683555414279302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,256,0.007497777541478475
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,1024,0.011379555695586734
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,64,0.0028728888266616394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,768,0.011695110963450538
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,128,0.004260444392760594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,7168,32,0.0022026666750510535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,128,0.009288888838556077
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,2048,0.018776888648668926
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,32,0.008975110948085785
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,512,0.011150221857759686
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,256,0.010081777969996134
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,12288,0.20661066638098824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,7168,64,0.008942222429646386
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,16384,0.30664534038967556
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,16384,0.17518488566080728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,12288,0.1344826618830363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,10240,0.17115555869208443
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,10240,0.11287555429670547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,8192,0.13697422875298396
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,7168,0.13571288850572374
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,6144,0.11664977338578965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,65536,0.6342319912380642
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,65536,1.0859706666734483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,5120,0.09752977556652492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,6144,0.06392177608278063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,8192,0.09247910976409912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,7168,0.08106578058666654
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,3072,0.05550133188565572
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,5120,0.058931556012895375
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,4096,0.06958399878607856
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,4096,0.0445493327246772
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,3072,0.03871022330390082
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,2560,0.04660177893108792
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,3584,0.04418755571047465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,2560,0.021379555265108745
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,3584,0.06866221957736544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,2048,0.03747289048300849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,1536,0.02871733241611057
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,768,0.005614222337802251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,1024,0.019727999965349834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,512,0.004935111022657818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,1536,0.01186222251918581
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,128,0.00371111101574368
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,1024,0.011235555840863122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,768,0.011312888728247749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,2048,0.012327999704413943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,256,0.009628444082207149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,256,0.004934222333961063
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,128,0.008765333228641087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,32,0.0021546667234765156
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,6144,64,0.0026577777332729767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,32,0.008750222623348236
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,64,0.009036444127559662
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,6144,512,0.01091733326514562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,12288,0.17308444446987578
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,12288,0.10383199983172947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,16384,0.22742666138543022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,16384,0.1456177764468723
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,10240,0.15563378069135878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,8192,0.11462310949961345
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,10240,0.09544799725214641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,6144,0.08699822425842285
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,7168,0.10090844498740302
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,8192,0.07048266463809542
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,7168,0.06260266568925646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,65536,0.48496802647908527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,5120,0.07273866732915242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,4096,0.05831199884414673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,3584,0.054607109891043774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,6144,0.05423644516203138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,4096,0.03843999902407328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,3072,0.04708622230423821
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,2560,0.012810666528013019
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,5120,0.05047466688685947
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,3584,0.034062223301993474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,2048,0.031833777825037636
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,1536,0.024409777588314478
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,65536,0.9087538189358182
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,2560,0.04440977838304308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,3072,0.02056711084312863
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,1024,0.016889777448442247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,2048,0.011924444801277585
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,768,0.005510222166776657
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,512,0.004791111167934206
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,256,0.005681777579916849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,1536,0.011706666813956367
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,1024,0.010685332947307162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,768,0.011074666347768573
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,64,0.0025235555238193935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,512,0.010439111126793755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,128,0.003075555587808291
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,256,0.009491555392742157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,5120,32,0.0021262222694026097
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,128,0.008814222282833522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,64,0.008560000194443597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,5120,32,0.008661333057615492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,12288,0.13946755727132162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,12288,0.08499377965927124
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,16384,0.10860000054041545
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,10240,0.11501510938008626
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,16384,0.2058124409781562
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,10240,0.0701120032204522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,8192,0.0925297803348965
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,7168,0.08138755295011732
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,8192,0.05729155408011543
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,65536,0.3910791079203288
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,6144,0.07021333111657037
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,7168,0.05526044302516513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,4096,0.05009688933690389
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,65536,0.7261315451727973
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,4096,0.023005333211686876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,6144,0.04420533445146349
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,5120,0.05870755513509115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,3584,0.013407111167907715
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,5120,0.038463112380769514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,2560,0.03224622209866842
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,3584,0.04968000120586819
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,1536,0.020012444920010038
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,2048,0.025928000609079998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,2560,0.01219555570019616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,3072,0.04313777883847555
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,1024,0.01399466726515028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,2048,0.011400889191362592
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,768,0.005096000101831225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,3072,0.012404444317022959
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,512,0.004048000193304486
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,256,0.0047493332790003884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,1536,0.011123555401961008
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,512,0.010287111004193624
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,128,0.0024622221373849446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,768,0.010571555958853828
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,64,0.0023813332534498638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,4096,32,0.0020737778395414352
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,1024,0.010236444572607676
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,128,0.008345777789751688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,256,0.00925511121749878
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,64,0.00831288927131229
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,4096,32,0.008198222352398766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,12288,0.1226044495900472
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,16384,0.16034221649169922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,16384,0.0962488916185167
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,12288,0.0742595526907179
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,10240,0.10104088650809394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,8192,0.08114311430189344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,65536,0.3423413435618083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,8192,0.050628443559010826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,7168,0.07158578104443021
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,10240,0.062331557273864746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,6144,0.06163555383682251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,3584,0.03977422250641717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,7168,0.04494755466779073
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,65536,0.6381315655178493
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,5120,0.03350399931271871
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,4096,0.013418667018413544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,4096,0.04464800159136454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,5120,0.05799733267890083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,3072,0.034120000070995755
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,2560,0.02904711167017619
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,2048,0.009082666701740688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,6144,0.04096800088882446
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,3072,0.01219555570019616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,1024,0.005614222337802251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,1536,0.0181688881582684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,2560,0.01202755586968528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,2048,0.0111191107167138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,1536,0.011002666420406766
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,512,0.006320000108745363
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,1024,0.010028444230556488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,256,0.0041928887367248535
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,3584,0.0129457778400845
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,768,0.010484444598356882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,128,0.002448888909485605
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,512,0.009911999934249455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,64,0.002313777804374695
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,768,0.005053333524200651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,256,0.008967999782827165
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,128,0.008267555799749162
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3584,32,0.0020906666500700847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,64,0.008228444390826756
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3584,32,0.008004444340864817
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,16384,0.1375057829750909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,12288,0.10579288668102688
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,16384,0.08296355273988512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,10240,0.08675822284486558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,12288,0.06543644269307454
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,8192,0.06981422503789265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,10240,0.054142223464118115
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,8192,0.04405155446794298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,7168,0.06163555383682251
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,6144,0.05318399932649401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,65536,0.3084533214569092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,5120,0.047204444805781044
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,6144,0.033223999871148005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,65536,0.5441875457763672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,7168,0.03976888789070977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,3584,0.03365866674317254
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,3072,0.02921600143114726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,2560,0.024713777833514746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,5120,0.017297777864668105
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,3584,0.012815111213260226
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,3072,0.012106666962305704
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,2048,0.0200311111079322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,1536,0.01572088897228241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,4096,0.03784444597032335
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,4096,0.013023999830087027
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,1024,0.0052693333062860705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,768,0.004954666727119022
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,2048,0.011052444577217102
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,512,0.005161777966552311
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,1536,0.010853333605660332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,256,0.003768889026509391
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,1024,0.010103111465771994
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,2560,0.011890666352377998
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,768,0.010271111296282874
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,128,0.0022524444179402455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,512,0.009782222410043081
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,64,0.0023013332651721104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,3072,32,0.0020666666742828158
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,256,0.008799999952316284
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,128,0.007919111185603673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,32,0.008074666890833113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,3072,64,0.007923555870850882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,16384,0.11628444989522298
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,12288,0.089629332224528
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,16384,0.0698453320397271
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,10240,0.07382222016652425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,12288,0.055789331595102944
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,65536,0.24981599383884004
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,8192,0.05929333633846707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,8192,0.03824888997607761
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,10240,0.045574221346113414
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,6144,0.0475466681851281
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,65536,0.45427465438842773
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,5120,0.0394720004664527
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,7168,0.05225155419773526
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,4096,0.031679110394583807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,7168,0.033462223079469465
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,3584,0.028199109766218398
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,3072,0.024570667081409033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,5120,0.013921777407328287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,6144,0.016520889268981088
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,2560,0.020788444413079154
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,3584,0.012632889052232107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,3072,0.011968000067604912
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,1536,0.013048888908492194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,2048,0.016618667377365958
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,1024,0.009504000345865885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,4096,0.012856889102194043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,2048,0.01088355564408832
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,1536,0.010737777584128909
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,2560,0.011852444046073489
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,768,0.00480355570713679
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,1024,0.010034666293197209
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,512,0.005011555635266834
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,256,0.003400000019205941
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,128,0.0021688888470331826
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,512,0.009631111390060848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,64,0.0021955555097924336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2560,32,0.002056888822052214
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,128,0.008046222229798635
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,64,0.007846222155623967
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,256,0.008861333131790161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,32,0.00777599960565567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2560,768,0.010254222485754225
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,16384,0.09316711293326484
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,16384,0.057568887869517006
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,12288,0.07251022259394328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,10240,0.05904710955089993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,12288,0.04625511169433594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,65536,0.20363733503553602
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,10240,0.03831733266512553
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,8192,0.04778933193948534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,7168,0.042381332980261914
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,5120,0.031192888816197712
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,6144,0.03749955693880717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,8192,0.021272889441914026
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,65536,0.36353866259257
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,6144,0.014789332946141561
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,4096,0.011507555842399597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,7168,0.016062221593326993
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,3072,0.019627556204795837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,4096,0.012755555411179861
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,3584,0.012576888832781049
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,2560,0.016768000192112394
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,3584,0.009360888765917884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,5120,0.013847110999955071
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,3072,0.011810666984981961
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,2048,0.013622221847375235
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,1536,0.010695111420419482
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,1024,0.004596444467703502
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,2560,0.011635555161370171
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,512,0.0046471113132105935
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,2048,0.01073066641887029
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,1536,0.010648888846238455
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,768,0.004601777841647466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,256,0.0028737777223189673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,768,0.00996888925631841
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,128,0.0022311111291249595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,512,0.009681777821646797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,1024,0.009791110952695211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,32,0.0020186667227082783
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,2048,64,0.0022453332526816265
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,256,0.008604444563388824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,128,0.00794400026400884
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,64,0.007799111306667328
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,2048,32,0.007860444486141205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,16384,0.0751697752210829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,12288,0.05952799982494778
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,16384,0.04654933346642388
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,10240,0.047998223039839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,12288,0.03716355562210083
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,7168,0.03478488988346524
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,65536,0.15674044026268855
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,65536,0.2891644371880425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,10240,0.019659555620617337
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,8192,0.038168887297312416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,6144,0.02923111120859782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,5120,0.023119111855824787
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,4096,0.01978577839003669
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,7168,0.015653333730167814
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,6144,0.014683556225564746
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,3072,0.01516177753607432
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,4096,0.012695999609099494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,2560,0.012879110872745514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,3584,0.012541333006487953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,5120,0.013819555441538492
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,2048,0.010072888599501716
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,1536,0.005701333284378052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,2560,0.011588444312413534
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,8192,0.016511999898486666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,1024,0.003083555648724238
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,3072,0.011800888511869641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,3584,0.008830221990744272
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,768,0.0034479999707804788
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,512,0.0034311111602518293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,1536,0.010553778045707278
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,1024,0.009727111293209923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,256,0.0026568888376156488
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,128,0.002130666747689247
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,768,0.010042666561073726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,512,0.009549332989586724
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,32,0.002019555618365606
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,256,0.00867022242810991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1536,64,0.0021706666383478376
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,128,0.007982222570313348
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,64,0.007751999629868402
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,32,0.007824888659848107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,16384,0.051607112089792885
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1536,2048,0.01071733319097095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,16384,0.027012444204754297
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,12288,0.04028088847796122
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,10240,0.03329777717590332
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,8192,0.012813333008024426
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,65536,0.11435911390516494
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,65536,0.1945964495340983
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,12288,0.02051199972629547
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,7168,0.010004444254769219
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,6144,0.008919110728634728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,8192,0.01644088824590047
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,5120,0.016581333345837064
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,10240,0.018569777409235638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,7168,0.015614221493403116
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,6144,0.014707555373509726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,4096,0.014177777700954013
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,3584,0.012276444170210095
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,5120,0.013660444153679742
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,2560,0.008828444613350762
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,4096,0.012630222572220696
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,3584,0.01254755589697096
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,3072,0.007436444362004598
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,1536,0.0034266666819651923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,2048,0.005053333524200651
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,1024,0.0029084444459941653
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,3072,0.011762667033407422
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,2048,0.010639111200968424
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,768,0.0032213332338465583
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,1536,0.010515555739402771
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,2560,0.011447110937701331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,512,0.002817777709828483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,1024,0.009722666607962714
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,256,0.002519999941190084
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,128,0.002048888968096839
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,64,0.0020453333854675293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,768,0.010042666561073726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,1024,32,0.0020248889923095703
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,256,0.008535111116038429
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,512,0.009543110926946005
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,128,0.007850666840871176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,64,0.00794044468137953
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,16384,0.0369759996732076
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,1024,32,0.007834666305118138
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,12288,0.030800888935724895
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,65536,0.13767022556728786
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,16384,0.02423111100991567
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,10240,0.008728889127572378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,12288,0.02051555613676707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,7168,0.008607111043400234
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,8192,0.009120889008045197
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,65536,0.0995982223086887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,10240,0.01850844422976176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,6144,0.007624000310897827
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,5120,0.013184888495339287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,4096,0.008354666332403818
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,7168,0.015558222929636637
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,3584,0.010019555687904358
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,6144,0.01458577811717987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,3072,0.00904088881280687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,5120,0.013693333499961428
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,8192,0.016498666670587327
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,2560,0.007251555720965068
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,2048,0.006212444355090459
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,3584,0.01237066669596566
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,4096,0.012634666429625617
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,1536,0.004044444610675176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,1024,0.002733333243264092
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,768,0.002965333354141977
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,2048,0.0107004443804423
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,2560,0.011409777734014722
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,3072,0.011693333586057028
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,512,0.0026933333526055017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,256,0.0023128889087173673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,1536,0.010538666612572141
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,1024,0.009689778089523315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,128,0.0019431112127171622
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,768,0.009844444692134857
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,64,0.0019528888579871918
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,768,32,0.002017777827050951
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,512,0.009562667045328353
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,256,0.008532444636027018
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,128,0.007893333004580604
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,64,0.007954667011896769
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,16384,0.025934222671720717
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,768,32,0.007811555431948767
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,65536,0.09351199865341187
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,12288,0.007502222226725684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,10240,0.017323555217848886
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,8192,0.007172444628344641
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,16384,0.024175999893082514
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,65536,0.0881893303659227
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,12288,0.020454222957293194
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,7168,0.006990222467316522
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,6144,0.005984888722499211
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,10240,0.01843644513024224
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,8192,0.01643022232585483
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,5120,0.009126221968068017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,4096,0.009034666750166152
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,3584,0.003976888954639435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,3072,0.005722666780153911
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,6144,0.014567999376191033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,4096,0.012637333737479316
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,2560,0.004302222281694412
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,7168,0.015593777100245157
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,5120,0.013642667068375481
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,3584,0.012455110748608908
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,2048,0.0029075555503368378
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,1536,0.00286311118139161
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,3072,0.01163200040658315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,1024,0.0026293332791990707
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,768,0.0027919999427265594
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,2560,0.011411555939250521
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,512,0.0026497778793176017
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,2048,0.01055022246307797
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,256,0.0023173333870040043
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,1536,0.010447111394670276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,1024,0.009617778162161509
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,768,0.009857777920034196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,128,0.0019288888821999233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,64,0.0018533333722088074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,512,32,0.0020293332636356354
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,512,0.0096142225795322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,256,0.0087013335691558
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,128,0.007993777592976889
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,64,0.007801777786678738
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,16384,0.006795555353164673
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,512,32,0.007827555967701806
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,65536,0.04856355653868782
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,12288,0.006301333506902059
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,10240,0.0052586665583981415
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,8192,0.004846222284767362
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,16384,0.024199111594094172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,7168,0.00905600024594201
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,65536,0.07476177480485705
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,12288,0.020414221617910597
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,6144,0.009327111144860586
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,5120,0.005943999936183293
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,10240,0.01848444508181678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,4096,0.0033608888172441055
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,8192,0.016456888781653512
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,7168,0.015517334143320719
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,3584,0.003399111123548614
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,6144,0.01460355520248413
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,3072,0.003128000017669466
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,5120,0.01368533323208491
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,2560,0.0029279999434947968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,2048,0.002477333363559511
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,1536,0.0024577778660588795
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,4096,0.012568888564904531
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,3584,0.0123848890264829
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,3072,0.011619555453459421
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,2560,0.011426666544543372
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,1024,0.002201777779393726
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,768,0.0023502223193645477
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,2048,0.01074044406414032
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,512,0.002324444345302052
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,256,0.002040888907180892
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,1536,0.010447111394670276
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,1024,0.009670221971140968
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,128,0.0017591110534138149
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,64,0.0017324444941348499
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,768,0.009810667071077559
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,512,0.009548444714811113
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,256,32,0.001997333433892992
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,256,0.008700444466537898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,128,0.00796088907453749
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,16384,0.005003555367390315
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,65536,0.0258577780591117
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,64,0.007850666840871176
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,256,32,0.007781333393520779
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,10240,0.0047022220161226065
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,12288,0.005341333233647876
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,8192,0.006393777827421824
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,65536,0.07075733608669706
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,16384,0.024199111594094172
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,7168,0.006322666588756774
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,6144,0.0063279999627007385
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,5120,0.0037813331517908308
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,12288,0.02033333314789666
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,10240,0.0185199992524253
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,8192,0.01644711030854119
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,7168,0.01551999979548984
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,3584,0.0030942221896515954
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,6144,0.01464000013139513
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,3072,0.002967111145456632
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,5120,0.013627555635240344
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,4096,0.012539555629094442
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,4096,0.0030577776746617425
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,2560,0.0026942222482628296
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,3584,0.012327111429638334
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,2048,0.0022444443570242985
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,1536,0.0022577777918842104
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,1024,0.0019964445382356644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,3072,0.01165244479974111
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,768,0.002242666565709644
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,2560,0.011392888923486074
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,512,0.0021893332401911416
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,2048,0.010575110713640848
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,1536,0.010421333213647207
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,1024,0.009502222140630087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,256,0.001954666649301847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,128,0.0016871111260520087
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,768,0.00980000032318963
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,64,0.0016746666903297107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,128,32,0.001954666649301847
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,512,0.009555555880069733
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,65536,0.007516444557242923
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,256,0.008656000097592672
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,128,0.007975111405054728
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,16384,0.005223999834722943
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,64,0.007819555699825287
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,12288,0.0047484445903036325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,128,32,0.007770666645632849
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,10240,0.004480000171396467
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,8192,0.005142222262091107
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,65536,0.07067110803392199
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,16384,0.024117334021462336
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,7168,0.005044444567627377
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,12288,0.02037244372897678
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,10240,0.018387556076049805
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,6144,0.005041777673694822
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,8192,0.01645066671901279
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,5120,0.0035075553589397003
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,4096,0.0029120000286234748
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,7168,0.015415110521846347
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,3584,0.0029884444342719186
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,6144,0.014555555250909595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,3072,0.0027502222607533136
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,5120,0.013581333061059317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,4096,0.012560000022252401
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,2560,0.0025928889711697898
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,2048,0.002157333410448498
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,3584,0.012326222327020435
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,1536,0.0021911110315057966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,3072,0.011682666838169098
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,1024,0.0019377778387731975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,2560,0.011301333705584208
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,768,0.0022168887986077205
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,2048,0.010608000059922537
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,1536,0.010393777655230628
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,256,0.0019413334214025075
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,1024,0.009649777577983009
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,512,0.0021031111892726687
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,128,0.00166311115026474
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,768,0.00980977796845966
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,512,0.009516444471147325
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,64,0.0016542221936914658
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,64,32,0.0019377778387731975
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,256,0.008549333446555668
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,65536,0.005068444543414646
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,128,0.007914666500356462
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,64,0.007749333149856991
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,16384,0.003666666646798452
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,64,32,0.007831999825106727
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,10240,0.0034835553831524322
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,12288,0.003544888976547453
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,65536,0.06952977842754789
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,16384,0.02382933265633053
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,8192,0.003953777667548922
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,7168,0.003925333420435588
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,12288,0.020111110475328233
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,6144,0.0038951109680864546
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,10240,0.018239999810854595
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,5120,0.0034844444857703317
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,4096,0.0028853333658642242
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,8192,0.016327111257447135
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,7168,0.01531555586391025
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,3584,0.0029813332690132987
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,3072,0.0027280000762807
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,6144,0.014387556248241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,2560,0.002575111058023241
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,5120,0.013463999662134381
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,2048,0.0021484444538752237
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,4096,0.012517333030700684
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,1536,0.00217333332531982
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,3584,0.012183999849690331
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,3072,0.011511111425028907
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,1024,0.0019368889431158702
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,768,0.002185777864522404
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,2560,0.01127911110719045
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,512,0.0021093332519133887
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,2048,0.010484444598356882
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,1536,0.0103502223889033
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,256,0.0019519999623298645
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,128,0.0016515555067194833
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,1024,0.00962933318482505
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,64,0.0016666666294137638
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,768,0.009857777920034196
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,float16,1,32,32,0.001863111017478837
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,512,0.009353777600659264
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,256,0.008599999878141616
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,128,0.007873777714040544
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,64,0.007598222129874759
VLLM,0.14.0,NVIDIA L40S,gemm,vllm_default,fp8,1,32,32,0.007860444486141205
